studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make video tiles be based on MatrixRTC member not LiveKit participants (#2701)

Browse Source 
* make tiles based on rtc member

* display missing lk participant + fix tile multiplier

* add show_non_member_participants config option

* per member tiles

* merge fixes

* linter

* linter and tests

* tests

* adapt tests (wip)

* Remove unused keys

* Fix optionality of nonMemberItemCount

* video is optional

* Mock RTC members

* Lint

* Merge fixes

* Fix user id

* Add explicit types for public fields

* isRTCParticipantAvailable => isLiveKitParticipantAvailable

* isLiveKitParticipantAvailable

* Readonly

* More keys removal

* Make local field based on view model class not observable

* Wording

* Fix RTC members in tes

* Tests again

* Lint

* Disable showing non-member tiles by default

* Duplicate screen sharing tiles like we used to

* Lint

* Revert function reordering

* Remove throttleTime from bad merge

* Cleanup

* Tidy config of show non-member settings

* tidy up handling of local rtc member in tests

* tidy up test init

* Fix mocks

* Cleanup

* Apply local override where participant not yet known

* Handle no visible media id

* Assertions for one-on-one view

* Remove isLiveKitParticipantAvailable and show via encryption status

* Handle no local media (yet)

* Remove unused effect for setting

* Tidy settings

* Avoid case of one-to-one layout with missing local or remote

* Iterate

* Remove option to show non-member tiles to simplify code review

* Remove unused code

* Remove more remnants of show-non-member-tiles

* iterate

* back

* Fix unit test

* Refactor

* Expose TestScheduler as global

* Fix incorrect type assertion

* Simplify speaking observer

* Fix

* Whitespace

* Make it clear that we are mocking MatrixRTC memberships

* Test case for only showing tiles for MatrixRTC session members

* Simplify diff

* Simplify diff

These changes are in https://github.com/element-hq/element-call/pull/2809

* .

* Whitespaces

* Use asObservable when exposing subject

* Show "waiting for media..." when no participant

* Additional test case

* Don't show "waiting for media..." in case of local participant

* Make the loading state more subtle
 - instead of a label we show a animated gradient

* Use correct key for matrix rtc foci in code comment. (#2838)

* Update src/tile/SpotlightTile.tsx

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Make the purpose of BaseMediaViewModel.local explicit

* Use named object instead of unnamed array for spotlightAndPip

* Refactor spotlightAndPip into spotlight and pip

* Use if statement instead of ternary for readability in spotlight and pip logic

* Review feedback

* Fix tests for CallEventAudioRenderer

* Lint

* Revert "Make the loading state more subtle"

This reverts commit 765f7b4f319b86839fcb4fde28d1e0604e542577.

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Fix spelling

* Remove a non-null assertion that failed at runtime

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
Co-authored-by: Hugh Nimmo-Smith <hughns@users.noreply.github.com>
This commit is contained in:
Timo
2024-12-06 12:28:37 +01:00
committed by GitHub
parent 21b62dbd89
commit 43c81a2758
16 changed files with 729 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

275
src/state/CallViewModel.ts
View File

@@ -18,12 +18,9 @@ import {
RemoteParticipant,
Track,
} from "livekit-client";
import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
import {
Room as MatrixRoom,
RoomMember,
RoomStateEvent,
} from "matrix-js-sdk/src/matrix";
import {
BehaviorSubject,
EMPTY,
Observable,
Subject,
@@ -49,6 +46,10 @@ import {
withLatestFrom,
} from "rxjs";
import { logger } from "matrix-js-sdk/src/logger";
import {
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { ViewModel } from "./ViewModel";
import {
@@ -222,41 +223,67 @@ interface LayoutScanState {
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker: Observable<boolean>;
public readonly presenter: Observable<boolean>;
public constructor(
public readonly id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.vm = participant.isLocal
? new LocalUserMediaViewModel(
id,
member,
participant as LocalParticipant,
encryptionSystem,
livekitRoom,
)
: new RemoteUserMediaViewModel(
id,
member,
participant as RemoteParticipant,
encryptionSystem,
livekitRoom,
);
this.participant = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
);
}
this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());
this.presenter = observeParticipantEvents(
participant,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
this.presenter = this.participant.pipe(
switchMap(
(p) =>
(p &&
observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled))) ??
of(false),
),
this.scope.state(),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant.next(newParticipant);
}
}
public destroy(): void {
@@ -267,6 +294,9 @@ class UserMedia {
class ScreenShare {
public readonly vm: ScreenShareViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
@@ -275,12 +305,15 @@ class ScreenShare {
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
participant,
this.participant.asObservable(),
encryptionSystem,
liveKitRoom,
participant.isLocal,
);
}
@@ -317,11 +350,11 @@ function findMatrixRoomMember(
export class CallViewModel extends ViewModel {
public readonly localVideo: Observable<LocalVideoTrack | null> =
observeTrackReference(
this.livekitRoom.localParticipant,
of(this.livekitRoom.localParticipant),
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef.publication?.track;
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
);
@@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value,
// Also react to changes in the list of members
fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
// Also react to changes in the MatrixRTC session list.
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null)),
]).pipe(
scan(
(
prevItems,
[remoteParticipants, { participant: localParticipant }, duplicateTiles],
[
remoteParticipants,
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
],
) => {
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const p of [localParticipant, ...remoteParticipants]) {
const id = p === localParticipant ? "local" : p.identity;
const member = findMatrixRoomMember(this.matrixRoom, id);
if (member === undefined)
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
);
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
const room = this.matrixRTCSession.room;
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
// Create as many tiles for this participant as called for by
// the duplicateTiles option
let participant:
| LocalParticipant
| RemoteParticipant
| undefined = undefined;
if (
rtcMember.sender === room.client.getUserId()! &&
rtcMember.deviceId === room.client.getDeviceId()
) {
livekitParticipantId = "local";
participant = localParticipant;
} else {
participant = remoteParticipants.find(
(p) => p.identity === livekitParticipantId,
);
}
const member = findMatrixRoomMember(room, livekitParticipantId);
if (!member) {
logger.error(
"Could not find member for media id: ",
livekitParticipantId,
);
}
for (let i = 0; i < 1 + duplicateTiles; i++) {
const userMediaId = `${id}:${i}`;
const indexedMediaId = `${livekitParticipantId}:${i}`;
const prevMedia = prevItems.get(indexedMediaId);
if (prevMedia && prevMedia instanceof UserMedia) {
prevMedia.updateParticipant(participant);
}
yield [
userMediaId,
prevItems.get(userMediaId) ??
indexedMediaId,
// We create UserMedia with or without a participant.
// This will be the initial value of a BehaviourSubject.
// Once a participant appears we will update the BehaviourSubject. (see above)
prevMedia ??
new UserMedia(
userMediaId,
indexedMediaId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
];
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
if (participant?.isScreenShareEnabled) {
const screenShareId = `${indexedMediaId}:screen-share`;
yield [
screenShareId,
prevItems.get(screenShareId) ??
new ScreenShare(
screenShareId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
@@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel {
}.bind(this)(),
);
for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
return newItems;
},
new Map<string, MediaItem>(),
@@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel {
),
);
private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
this.mediaItems.pipe(
map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
@@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
private readonly spotlightSpeaker: Observable<UserMediaViewModel | null> =
this.userMedia.pipe(
switchMap((mediaItems) =>
mediaItems.length === 0
@@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel {
),
),
),
scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
(prev, mediaItems) => {
// Only remote users that are still in the call should be sticky
const [stickyMedia, stickySpeaking] =
@@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel {
// Otherwise, spotlight an arbitrary remote user
mediaItems.find(([m]) => !m.vm.local)?.[0] ??
// Otherwise, spotlight the local user
mediaItems.find(([m]) => m.vm.local)![0]);
mediaItems.find(([m]) => m.vm.local)?.[0]);
},
null,
),
map((speaker) => speaker.vm),
map((speaker) => speaker?.vm ?? null),
this.scope.state(),
);
@@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightAndPip: Observable<
[Observable<MediaViewModel[]>, Observable<UserMediaViewModel | null>]
> = this.screenShares.pipe(
map((screenShares) =>
screenShares.length > 0
? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
: ([
this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
this.spotlightSpeaker.pipe(
switchMap((speaker) =>
speaker.local
? of(null)
: this.localUserMedia.pipe(
switchMap((vm) =>
vm.alwaysShow.pipe(
map((alwaysShow) => (alwaysShow ? vm : null)),
),
),
),
),
),
] as const),
),
);
private readonly spotlight: Observable<MediaViewModel[]> =
this.spotlightAndPip.pipe(
switchMap(([spotlight]) => spotlight),
this.screenShares.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0) {
return of(screenShares.map((m) => m.vm));
}
return this.spotlightSpeaker.pipe(
map((speaker) => (speaker ? [speaker] : [])),
);
}),
this.scope.state(),
);
private readonly pip: Observable<UserMediaViewModel | null> = combineLatest([
this.screenShares,
this.spotlightSpeaker,
this.mediaItems,
]).pipe(
switchMap(([screenShares, spotlight, mediaItems]) => {
if (screenShares.length > 0) {
return this.spotlightSpeaker;
}
if (!spotlight || spotlight.local) {
return of(null);
}
const localUserMedia = mediaItems.find(
(m) => m.vm instanceof LocalUserMediaViewModel,
) as UserMedia | undefined;
const localUserMediaViewModel = localUserMedia?.vm as
| LocalUserMediaViewModel
| undefined;
if (!localUserMediaViewModel) {
return of(null);
}
return localUserMediaViewModel.alwaysShow.pipe(
map((alwaysShow) => {
if (alwaysShow) {
return localUserMediaViewModel;
}
return null;
}),
);
}),
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
@@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel {
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
startWith(false),
);
@@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel {
this.mediaItems.pipe(
map((mediaItems) => {
if (mediaItems.length !== 2) return null;
const local = mediaItems.find((vm) => vm.vm.local)!
.vm as LocalUserMediaViewModel;
const local = mediaItems.find((vm) => vm.vm.local)?.vm as
| LocalUserMediaViewModel
| undefined;
const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as
| RemoteUserMediaViewModel
| undefined;
// There might not be a remote tile if there are screen shares, or if
// only the local user is in the call and they're using the duplicate
// tiles option
if (remote === undefined) return null;
if (!remote || !local) return null;
return { type: "one-on-one", local, remote };
}),
@@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel {
public constructor(
// A call is permanently tied to a single Matrix room and LiveKit room
private readonly matrixRoom: MatrixRoom,
private readonly matrixRTCSession: MatrixRTCSession,
private readonly livekitRoom: LivekitRoom,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState: Observable<ECConnectionState>,