studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace generateKeyed$ with a redesigned generateItems operator

Browse Source 
And use it to clean up a number of code smells, fix some reactivity bugs, and avoid some resource leaks.
This commit is contained in:
Robin
2025-11-07 17:36:16 -05:00
parent 1f386a1d57
commit b4c17ed26d
18 changed files with 610 additions and 441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
src/state/CallViewModel/CallViewModel.ts
View File

@@ -52,7 +52,7 @@ import {
ScreenShareViewModel,
type UserMediaViewModel,
} from "../MediaViewModel";
import { accumulate, generateKeyed$, pauseWhen } from "../../utils/observable";
import { accumulate, generateItems, pauseWhen } from "../../utils/observable";
import {
duplicateTiles,
MatrixRTCMode,
@@ -75,7 +75,7 @@ import {
} from "../../reactions";
import { shallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type Behavior, constant } from "../Behavior";
import { type Behavior } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
@@ -370,103 +370,101 @@ export class CallViewModel {
);
/**
* List of MediaItems that we want to have tiles for.
* List of user media (camera feeds) that we want tiles for.
*/
// TODO KEEP THIS!! and adapt it to what our membershipManger returns
// TODO this also needs the local participant to be added.
private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
generateKeyed$<
[typeof this.matrixLivekitMembers$.value, number],
MediaItem,
MediaItem[]
>(
private readonly userMedia$ = this.scope.behavior<UserMedia[]>(
combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]).pipe(
// Generate a collection of MediaItems from the list of expected (whether
// present or missing) LiveKit participants.
combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]),
([{ value: matrixLivekitMembers }, duplicateTiles], createOrGet) => {
const items: MediaItem[] = [];
for (const {
connection,
participant,
member,
displayName,
generateItems(
function* ([{ value: matrixLivekitMembers }, duplicateTiles]) {
for (const {
participantId,
userId,
participant$,
connection$,
displayName$,
mxcAvatarUrl$,
} of matrixLivekitMembers)
for (let dup = 0; dup < 1 + duplicateTiles; dup++)
yield {
keys: [
dup,
participantId,
userId,
participant$,
connection$,
displayName$,
mxcAvatarUrl$,
],
data: undefined,
};
},
(
scope,
_data$,
dup,
participantId,
} of matrixLivekitMembers) {
if (connection === undefined) {
logger.warn("connection is not yet initialised.");
continue;
}
for (let i = 0; i < 1 + duplicateTiles; i++) {
const mediaId = `${participantId}:${i}`;
const lkRoom = connection?.livekitRoom;
const url = connection?.transport.livekit_service_url;
userId,
participant$,
connection$,
displayName$,
mxcAvatarUrl$,
) => {
const livekitRoom$ = scope.behavior(
connection$.pipe(map((c) => c?.livekitRoom)),
);
const focusUrl$ = scope.behavior(
connection$.pipe(map((c) => c?.transport.livekit_service_url)),
);
const item = createOrGet(
mediaId,
(scope) =>
// We create UserMedia with or without a participant.
// This will be the initial value of a BehaviourSubject.
// Once a participant appears we will update the BehaviourSubject. (see below)
new UserMedia(
scope,
mediaId,
member,
participant,
this.options.encryptionSystem,
lkRoom,
url,
this.mediaDevices,
this.pretendToBeDisconnected$,
constant(displayName ?? "[👻]"),
this.handsRaised$.pipe(
map((v) => v[participantId]?.time ?? null),
),
this.reactions$.pipe(
map((v) => v[participantId] ?? undefined),
),
),
);
items.push(item);
(item as UserMedia).updateParticipant(participant);
if (participant?.isScreenShareEnabled) {
const screenShareId = `${mediaId}:screen-share`;
items.push(
createOrGet(
screenShareId,
(scope) =>
new ScreenShare(
scope,
screenShareId,
member,
participant,
this.options.encryptionSystem,
lkRoom,
url,
this.pretendToBeDisconnected$,
constant(displayName ?? "[👻]"),
),
),
);
}
}
}
return items;
},
return new UserMedia(
scope,
`${participantId}:${dup}`,
userId,
participant$,
this.options.encryptionSystem,
livekitRoom$,
focusUrl$,
this.mediaDevices,
this.pretendToBeDisconnected$,
displayName$,
mxcAvatarUrl$,
this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
this.reactions$.pipe(map((v) => v[participantId] ?? undefined)),
);
},
),
),
);
/**
* List of MediaItems that we want to display, that are of type UserMedia
* List of all media items (user media and screen share media) that we want
* tiles for.
*/
private readonly userMedia$ = this.scope.behavior<UserMedia[]>(
this.mediaItems$.pipe(
map((mediaItems) =>
mediaItems.filter((m): m is UserMedia => m instanceof UserMedia),
private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
this.userMedia$.pipe(
switchMap((userMedia) =>
userMedia.length === 0
? of([])
: combineLatest(
userMedia.map((m) => m.screenShares$),
(...screenShares) => [...userMedia, ...screenShares.flat(1)],
),
),
),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
private readonly screenShares$ = this.scope.behavior<ScreenShare[]>(
this.mediaItems$.pipe(
map((mediaItems) =>
mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare),
),
),
[],
);
public readonly joinSoundEffect$ = this.userMedia$.pipe(
@@ -544,17 +542,6 @@ export class CallViewModel {
tap((reason) => this.leaveHoisted$.next(reason)),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
private readonly screenShares$ = this.scope.behavior<ScreenShare[]>(
this.mediaItems$.pipe(
map((mediaItems) =>
mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare),
),
),
);
private readonly spotlightSpeaker$ =
this.scope.behavior<UserMediaViewModel | null>(
this.userMedia$.pipe(

20
src/state/CallViewModel/localMember/LocalMembership.ts
View File

@@ -5,7 +5,13 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LocalTrack, type E2EEOptions } from "livekit-client";
import {
type LocalTrack,
type E2EEOptions,
type Participant,
ParticipantEvent,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import {
type LivekitTransport,
type MatrixRTCSession,
@@ -26,11 +32,9 @@ import {
switchMap,
take,
takeWhile,
tap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../../ObservableScope";
@@ -521,3 +525,13 @@ export const createLocalMembership$ = ({
toggleScreenSharing,
};
};
export function observeSharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}

56
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -20,7 +20,7 @@ import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateKeyed$ } from "../../../utils/observable.ts";
import { generateItemsWithEpoch } from "../../../utils/observable.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
@@ -144,34 +144,32 @@ export function createConnectionManager$({
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
generateKeyed$<Epoch<LivekitTransport[]>, Connection, Epoch<Connection[]>>(
transports$,
(transports, createOrGet) => {
const createConnection =
(
transport: LivekitTransport,
): ((scope: ObservableScope) => Connection) =>
(scope) => {
const connection = connectionFactory.createConnection(
transport,
scope,
logger,
);
// Start the connection immediately
// Use connection state to track connection progress
void connection.start();
// TODO subscribe to connection state to retry or log issues?
return connection;
};
return transports.mapInner((transports) => {
return transports.map((transport) => {
const key =
transport.livekit_service_url + "|" + transport.livekit_alias;
return createOrGet(key, createConnection(transport));
});
});
},
transports$.pipe(
generateItemsWithEpoch(
function* (transports) {
for (const transport of transports)
yield {
keys: [transport.livekit_service_url, transport.livekit_alias],
data: undefined,
};
},
(scope, _data$, serviceUrl, alias) => {
const connection = connectionFactory.createConnection(
{
type: "livekit",
livekit_service_url: serviceUrl,
livekit_alias: alias,
},
scope,
logger,
);
// Start the connection immediately
// Use connection state to track connection progress
void connection.start();
// TODO subscribe to connection state to retry or log issues?
return connection;
},
),
),
);

95
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -16,32 +16,31 @@ import {
import { combineLatest, filter, map } from "rxjs";
// eslint-disable-next-line rxjs/no-internal
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
import { type Room as MatrixRoom } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "./ConnectionManager";
import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope";
import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
import { memberDisplaynames$ } from "./displayname";
import { type Connection } from "./Connection";
import { generateItemsWithEpoch } from "../../../utils/observable";
/**
* Represent a matrix call member and his associated livekit participation.
* Represents a Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface MatrixLivekitMember {
membership: CallMembership;
displayName?: string;
participant?: LocalLivekitParticipant | RemoteLivekitParticipant;
connection?: Connection;
/**
* TODO Try to remove this! Its waaay to much information.
* Just get the member's avatar
* @deprecated
*/
member: RoomMember;
mxcAvatarUrl?: string;
participantId: string;
userId: string;
membership$: Behavior<CallMembership>;
participant$: Behavior<
LocalLivekitParticipant | RemoteLivekitParticipant | null
>;
connection$: Behavior<Connection | undefined>;
displayName$: Behavior<string>;
mxcAvatarUrl$: Behavior<string | undefined>;
}
interface Props {
@@ -100,44 +99,54 @@ export function createMatrixLivekitMembers$({
{ value: membershipsWithTransports, epoch },
{ value: managerData },
{ value: displaynames },
]) => {
const items: MatrixLivekitMember[] = membershipsWithTransports.map(
({ membership, transport }) => {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
]) =>
new Epoch(
[membershipsWithTransports, managerData, displaynames] as const,
epoch,
),
),
generateItemsWithEpoch(
function* ([membershipsWithTransports, managerData, displaynames]) {
for (const { membership, transport } of membershipsWithTransports) {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport
? managerData.getParticipantForTransport(transport)
: [];
const participant = participants.find(
(p) => p.identity == participantId,
);
const member = getRoomMemberFromRtcMember(
const participants = transport
? managerData.getParticipantForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;
// This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
const member = matrixRoom.getMember(membership.userId);
const connection = transport
? managerData.getConnectionForTransport(transport)
: undefined;
let displayName = displaynames.get(membership.userId);
if (displayName === undefined) {
logger.warn(`No display name for user ${membership.userId}`);
displayName = "";
}
yield {
keys: [participantId, membership.userId],
data: {
membership,
matrixRoom,
)?.member;
const connection = transport
? managerData.getConnectionForTransport(transport)
: undefined;
const displayName = displaynames.get(participantId);
return {
participant,
membership,
connection,
// This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
// TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely.
member: member as RoomMember,
displayName,
mxcAvatarUrl: member?.getMxcAvatarUrl(),
participantId,
};
},
);
return new Epoch(items, epoch);
},
};
}
},
(scope, data$, participantId, userId) => ({
participantId,
userId,
...scope.splitBehavior(data$),
}),
),
),
// new Epoch([]),
);
}

56
src/state/CallViewModel/remoteMembers/displayname.test.ts
View File

@@ -97,7 +97,7 @@ test.skip("should always have our own user", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", {
a: new Map<string, string>([
["@local:example.com:DEVICE000", "@local:example.com"],
["@local:example.com", "@local:example.com"],
]),
});
});
@@ -130,9 +130,9 @@ test("should get displayName for users", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@alice:example.com:DEVICE1", "Alice"],
["@bob:example.com:DEVICE1", "Bob"],
// ["@local:example.com", "it's a me"],
["@alice:example.com", "Alice"],
["@bob:example.com", "Bob"],
]),
});
});
@@ -152,8 +152,8 @@ test("should use userId if no display name", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@no-name:foo.bar:D000", "@no-name:foo.bar"],
// ["@local:example.com", "it's a me"],
["@no-name:foo.bar", "@no-name:foo.bar"],
]),
});
});
@@ -179,12 +179,12 @@ test("should disambiguate users with same display name", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
["@bob:example.com:DEVICE2", "Bob (@bob:example.com)"],
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
["@carl:example.com:C000", "Carl (@carl:example.com)"],
["@evil:example.com:E000", "Carl (@evil:example.com)"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
["@carl:example.com", "Carl (@carl:example.com)"],
["@evil:example.com", "Carl (@evil:example.com)"],
]),
});
});
@@ -208,13 +208,13 @@ test("should disambiguate when needed", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:DEVICE1", "Bob"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob"],
]),
b: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
});
});
@@ -238,13 +238,13 @@ test.skip("should keep disambiguated name when other leave", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
b: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
]),
});
});
@@ -273,14 +273,14 @@ test("should disambiguate on name change", () => {
expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", {
a: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:B000", "Bob"],
["@carl:example.com:C000", "Carl"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob"],
["@carl:example.com", "Carl"],
]),
b: new Map<string, string>([
// ["@local:example.com:DEVICE000", "it's a me"],
["@bob:example.com:B000", "Bob (@bob:example.com)"],
["@carl:example.com:C000", "Bob (@carl:example.com)"],
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@carl:example.com", "Bob (@carl:example.com)"],
]),
});
});

25
src/state/CallViewModel/remoteMembers/displayname.ts
View File

@@ -42,7 +42,7 @@ export function createRoomMembers$(
* any displayname that clashes with another member. Only members
* joined to the call are considered here.
*
* @returns Map<member.id, displayname> uses the rtc member idenitfier as the key.
* @returns Map<userId, displayname> uses the Matrix user ID as the key.
*/
// don't do this work more times than we need to. This is achieved by converting to a behavior:
export const memberDisplaynames$ = (
@@ -66,19 +66,14 @@ export const memberDisplaynames$ = (
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
// TODO a hard-coded participant ID ? should use rtcMember.membershipID instead?
const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`;
const { member } = getRoomMemberFromRtcMember(rtcMember, room);
if (!member) {
logger.error(
"Could not find member for participant id:",
matrixIdentifier,
);
const member = room.getMember(rtcMember.userId);
if (member === null) {
logger.error(`Could not find member for user ${rtcMember.userId}`);
continue;
}
const disambiguate = shouldDisambiguate(member, memberships, room);
displaynameMap.set(
matrixIdentifier,
rtcMember.userId,
calculateDisplayName(member, disambiguate),
);
}
@@ -87,13 +82,3 @@ export const memberDisplaynames$ = (
),
new Epoch(new Map<string, string>()),
);
export function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: Pick<MatrixRoom, "getMember">,
): { id: string; member: RoomMember | undefined } {
return {
id: rtcMember.userId + ":" + rtcMember.deviceId,
member: room.getMember(rtcMember.userId) ?? undefined,
};
}

123
src/state/MediaViewModel.ts
View File

@@ -27,7 +27,6 @@ import {
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
@@ -44,6 +43,7 @@ import {
startWith,
switchMap,
throttleTime,
distinctUntilChanged,
} from "rxjs";
import { alwaysShowSelf } from "../settings/settings";
@@ -180,29 +180,35 @@ function observeRemoteTrackReceivingOkay$(
}
function encryptionErrorObservable$(
room: LivekitRoom,
room$: Behavior<LivekitRoom | undefined>,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
): Observable<boolean> {
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return room$.pipe(
switchMap((room) => {
if (room === undefined) return of(false);
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
return false;
}),
);
}),
distinctUntilChanged(),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
@@ -250,11 +256,9 @@ abstract class BaseMediaViewModel {
*/
public readonly id: string,
/**
* The Matrix room member to which this media belongs.
* The Matrix user to which this media belongs.
*/
// TODO: Fully separate the data layer from the UI layer by keeping the
// member object internal
public readonly member: RoomMember,
public readonly userId: string,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant$: Observable<
@@ -264,9 +268,10 @@ abstract class BaseMediaViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
public readonly focusURL: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
public readonly focusUrl$: Behavior<string | undefined>,
public readonly displayName$: Behavior<string>,
public readonly mxcAvatarUrl$: Behavior<string | undefined>,
) {
const audio$ = this.observeTrackReference$(audioSource);
this.video$ = this.observeTrackReference$(videoSource);
@@ -294,13 +299,13 @@ abstract class BaseMediaViewModel {
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return combineLatest([
encryptionErrorObservable$(
livekitRoom,
livekitRoom$,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable$(
livekitRoom,
livekitRoom$,
participant,
encryptionSystem,
"InvalidKey",
@@ -320,7 +325,7 @@ abstract class BaseMediaViewModel {
} else {
return combineLatest([
encryptionErrorObservable$(
livekitRoom,
livekitRoom$,
participant,
encryptionSystem,
"InvalidKey",
@@ -402,26 +407,28 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
member: RoomMember,
userId: string,
participant$: Observable<LocalParticipant | RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
public readonly handRaised$: Behavior<Date | null>,
public readonly reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
member,
userId,
participant$,
encryptionSystem,
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
focusUrl,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
);
const media$ = this.scope.behavior(
@@ -538,25 +545,27 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
member: RoomMember,
userId: string,
participant$: Behavior<LocalParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusURL: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
private readonly mediaDevices: MediaDevices,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
handRaised$: Behavior<Date | null>,
reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
member,
userId,
participant$,
encryptionSystem,
livekitRoom,
focusURL,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
handRaised$,
reaction$,
);
@@ -648,25 +657,27 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
member: RoomMember,
userId: string,
participant$: Observable<RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
handRaised$: Behavior<Date | null>,
reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
member,
userId,
participant$,
encryptionSystem,
livekitRoom,
focusUrl,
displayname$,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
handRaised$,
reaction$,
);
@@ -747,26 +758,28 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
member: RoomMember,
userId: string,
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
public readonly local: boolean,
) {
super(
scope,
id,
member,
userId,
participant$,
encryptionSystem,
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
focusUrl,
displayname$,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
);
}
}

43
src/state/ObservableScope.ts
View File

@@ -24,7 +24,11 @@ import { type Behavior } from "./Behavior";
type MonoTypeOperator = <T>(o: Observable<T>) => Observable<T>;
export const noInitialValue = Symbol("nothing");
type SplitBehavior<T> = keyof T extends string | number
? { [K in keyof T as `${K}$`]: Behavior<T[K]> }
: never;
const nothing = Symbol("nothing");
/**
* A scope which limits the execution lifetime of its bound Observables.
@@ -59,7 +63,10 @@ export class ObservableScope {
* Converts an Observable to a Behavior. If no initial value is specified, the
* Observable must synchronously emit an initial value.
*/
public behavior<T>(setValue$: Observable<T>, initialValue?: T): Behavior<T> {
public behavior<T>(
setValue$: Observable<T>,
initialValue: T | typeof nothing = nothing,
): Behavior<T> {
const subject$ = new BehaviorSubject(initialValue);
// Push values from the Observable into the BehaviorSubject.
// BehaviorSubjects have an undesirable feature where if you call 'complete',
@@ -74,7 +81,7 @@ export class ObservableScope {
subject$.error(err);
},
});
if (subject$.value === noInitialValue)
if (subject$.value === nothing)
throw new Error("Behavior failed to synchronously emit an initial value");
return subject$ as Behavior<T>;
}
@@ -115,27 +122,27 @@ export class ObservableScope {
value$: Behavior<T>,
callback: (value: T) => Promise<(() => Promise<void>) | void>,
): void {
let latestValue: T | typeof noInitialValue = noInitialValue;
let reconciledValue: T | typeof noInitialValue = noInitialValue;
let latestValue: T | typeof nothing = nothing;
let reconciledValue: T | typeof nothing = nothing;
let cleanUp: (() => Promise<void>) | void = undefined;
value$
.pipe(
catchError(() => EMPTY), // Ignore errors
this.bind(), // Limit to the duration of the scope
endWith(noInitialValue), // Clean up when the scope ends
endWith(nothing), // Clean up when the scope ends
)
.subscribe((value) => {
void (async (): Promise<void> => {
if (latestValue === noInitialValue) {
if (latestValue === nothing) {
latestValue = value;
while (latestValue !== reconciledValue) {
await cleanUp?.(); // Call the previous value's clean-up handler
reconciledValue = latestValue;
if (latestValue !== noInitialValue)
if (latestValue !== nothing)
cleanUp = await callback(latestValue); // Sync current value
}
// Reset to signal that reconciliation is done for now
latestValue = noInitialValue;
latestValue = nothing;
} else {
// There's already an instance of the above 'while' loop running
// concurrently. Just update the latest value and let it be handled.
@@ -144,6 +151,24 @@ export class ObservableScope {
})();
});
}
/**
* Splits a Behavior of objects with static properties into an object with
* Behavior properties.
*
* For example, splitting a Behavior<{ name: string, age: number }> results in
* an object of type { name$: Behavior<string> age$: Behavior<number> }.
*/
public splitBehavior<T extends object>(
input$: Behavior<T>,
): SplitBehavior<T> {
return Object.fromEntries(
Object.keys(input$.value).map((key) => [
`${key}$`,
this.behavior(input$.pipe(map((input) => input[key as keyof T]))),
]),
) as SplitBehavior<T>;
}
}
/**

21
src/state/ScreenShare.ts
View File

@@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { of, type Observable } from "rxjs";
import { of } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
@@ -13,7 +13,6 @@ import {
import { type ObservableScope } from "./ObservableScope.ts";
import { ScreenShareViewModel } from "./MediaViewModel.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { Behavior } from "./Behavior.ts";
@@ -28,24 +27,26 @@ export class ScreenShare {
public constructor(
private readonly scope: ObservableScope,
id: string,
member: RoomMember,
userId: string,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Observable<string>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
) {
this.vm = new ScreenShareViewModel(
this.scope,
id,
member,
userId,
of(participant),
encryptionSystem,
livekitRoom,
focusUrl,
livekitRoom$,
focusUrl$,
pretendToBeDisconnected$,
this.scope.behavior(displayName$),
displayName$,
mxcAvatarUrl$,
participant.isLocal,
);
}

8
src/state/TileStore.ts
View File

@@ -14,7 +14,7 @@ import { fillGaps } from "../utils/iter";
import { debugTileLayout } from "../settings/settings";
function debugEntries(entries: GridTileData[]): string[] {
return entries.map((e) => e.media.member?.rawDisplayName ?? "[👻]");
return entries.map((e) => e.media.displayName$.value);
}
let DEBUG_ENABLED = false;
@@ -156,7 +156,7 @@ export class TileStoreBuilder {
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.member?.rawDisplayName ?? "[👻]")}`,
`[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.displayName$.value)}`,
);
if (this.spotlight !== null) throw new Error("Spotlight already set");
@@ -180,7 +180,7 @@ export class TileStoreBuilder {
public registerGridTile(media: UserMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register grid tile: ${media.member?.rawDisplayName ?? "[👻]"}`,
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
);
if (this.spotlight !== null) {
@@ -263,7 +263,7 @@ export class TileStoreBuilder {
public registerPipTile(media: UserMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register PiP tile: ${media.member?.rawDisplayName ?? "[👻]"}`,
`[TileStore, ${this.generation}] register PiP tile: ${media.displayName$.value}`,
);
// If there is a single grid tile that we can reuse

117
src/state/UserMedia.ts
View File

@@ -5,17 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
import {
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
@@ -29,11 +21,12 @@ import {
type UserMediaViewModel,
} from "./MediaViewModel.ts";
import type { Behavior } from "./Behavior.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { MediaDevices } from "./MediaDevices.ts";
import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
import { generateItems } from "../utils/observable.ts";
import { ScreenShare } from "./ScreenShare.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
@@ -72,35 +65,35 @@ enum SortingBin {
/**
* A user media item to be presented in a tile. This is a thin wrapper around
* UserMediaViewModel which additionally determines the media item's sorting bin
* for inclusion in the call layout.
* for inclusion in the call layout and tracks associated screen shares.
*/
export class UserMedia {
private readonly participant$ = new BehaviorSubject(this.initialParticipant);
public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.member,
this.userId,
this.participant$ as Behavior<LocalParticipant | null>,
this.encryptionSystem,
this.livekitRoom,
this.focusURL,
this.livekitRoom$,
this.focusUrl$,
this.mediaDevices,
this.scope.behavior(this.displayname$),
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.member,
this.userId,
this.participant$ as Behavior<RemoteParticipant | null>,
this.encryptionSystem,
this.livekitRoom,
this.focusURL,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.scope.behavior(this.displayname$),
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
@@ -109,12 +102,55 @@ export class UserMedia {
observeSpeaker$(this.vm.speaking$),
);
private readonly presenter$ = this.scope.behavior(
/**
* All screen share media associated with this user media.
*/
public readonly screenShares$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (p === null ? of(false) : sharingScreen$(p))),
switchMap((p) =>
p === null
? of([])
: observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(
// Technically more than one screen share might be possible... our
// MediaViewModels don't support it though since they look for a unique
// track for the given source. So generateItems here is a bit overkill.
generateItems(
function* (p) {
if (p.isScreenShareEnabled)
yield {
keys: ["screen-share"],
data: undefined,
};
},
(scope, _data$, key) =>
new ScreenShare(
scope,
`${this.id}:${key}`,
this.userId,
p,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
),
),
),
),
),
);
private readonly presenter$ = this.scope.behavior(
this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)),
);
/**
* Which sorting bin the media item should be placed in.
*/
@@ -147,37 +183,18 @@ export class UserMedia {
public constructor(
private readonly scope: ObservableScope,
public readonly id: string,
private readonly member: RoomMember,
private readonly initialParticipant:
| LocalParticipant
| RemoteParticipant
| null = null,
private readonly userId: string,
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
>,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom: LivekitRoom,
private readonly focusURL: string,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
private readonly focusUrl$: Behavior<string | undefined>,
private readonly mediaDevices: MediaDevices,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
private readonly displayname$: Observable<string>,
private readonly displayName$: Behavior<string>,
private readonly mxcAvatarUrl$: Behavior<string | undefined>,
private readonly handRaised$: Observable<Date | null>,
private readonly reaction$: Observable<ReactionOption | null>,
) {}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | null = null,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
}
export function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}