studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make video tiles be based on MatrixRTC member not LiveKit participants (#2701)

Browse Source 
* make tiles based on rtc member

* display missing lk participant + fix tile multiplier

* add show_non_member_participants config option

* per member tiles

* merge fixes

* linter

* linter and tests

* tests

* adapt tests (wip)

* Remove unused keys

* Fix optionality of nonMemberItemCount

* video is optional

* Mock RTC members

* Lint

* Merge fixes

* Fix user id

* Add explicit types for public fields

* isRTCParticipantAvailable => isLiveKitParticipantAvailable

* isLiveKitParticipantAvailable

* Readonly

* More keys removal

* Make local field based on view model class not observable

* Wording

* Fix RTC members in tes

* Tests again

* Lint

* Disable showing non-member tiles by default

* Duplicate screen sharing tiles like we used to

* Lint

* Revert function reordering

* Remove throttleTime from bad merge

* Cleanup

* Tidy config of show non-member settings

* tidy up handling of local rtc member in tests

* tidy up test init

* Fix mocks

* Cleanup

* Apply local override where participant not yet known

* Handle no visible media id

* Assertions for one-on-one view

* Remove isLiveKitParticipantAvailable and show via encryption status

* Handle no local media (yet)

* Remove unused effect for setting

* Tidy settings

* Avoid case of one-to-one layout with missing local or remote

* Iterate

* Remove option to show non-member tiles to simplify code review

* Remove unused code

* Remove more remnants of show-non-member-tiles

* iterate

* back

* Fix unit test

* Refactor

* Expose TestScheduler as global

* Fix incorrect type assertion

* Simplify speaking observer

* Fix

* Whitespace

* Make it clear that we are mocking MatrixRTC memberships

* Test case for only showing tiles for MatrixRTC session members

* Simplify diff

* Simplify diff

These changes are in https://github.com/element-hq/element-call/pull/2809

* .

* Whitespaces

* Use asObservable when exposing subject

* Show "waiting for media..." when no participant

* Additional test case

* Don't show "waiting for media..." in case of local participant

* Make the loading state more subtle
 - instead of a label we show a animated gradient

* Use correct key for matrix rtc foci in code comment. (#2838)

* Update src/tile/SpotlightTile.tsx

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Make the purpose of BaseMediaViewModel.local explicit

* Use named object instead of unnamed array for spotlightAndPip

* Refactor spotlightAndPip into spotlight and pip

* Use if statement instead of ternary for readability in spotlight and pip logic

* Review feedback

* Fix tests for CallEventAudioRenderer

* Lint

* Revert "Make the loading state more subtle"

This reverts commit 765f7b4f319b86839fcb4fde28d1e0604e542577.

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Fix spelling

* Remove a non-null assertion that failed at runtime

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
Co-authored-by: Hugh Nimmo-Smith <hughns@users.noreply.github.com>
This commit is contained in:
Timo
2024-12-06 12:28:37 +01:00
committed by GitHub
parent 21b62dbd89
commit 43c81a2758
16 changed files with 729 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
src/state/CallViewModel.test.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { test, vi, onTestFinished, it } from "vitest";
import {
combineLatest,
debounceTime,
@@ -25,6 +25,7 @@ import {
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import { isEqual } from "lodash-es";
import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -34,6 +35,8 @@ import {
mockMatrixRoomMember,
mockRemoteParticipant,
withTestScheduler,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import {
ECAddonConnectionState,
@@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType";
vi.mock("@livekit/components-core");
const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
const carol = mockMatrixRoomMember({ userId: "@carol:example.org" });
const dave = mockMatrixRoomMember({ userId: "@dave:example.org" });
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const aliceId = `${alice.userId}:AAAA`;
const bobId = `${bob.userId}:BBBB`;
const daveId = `${dave.userId}:DDDD`;
const alice = mockMatrixRoomMember(aliceRtcMember);
const bob = mockMatrixRoomMember(bobRtcMember);
const carol = mockMatrixRoomMember(localRtcMember);
const dave = mockMatrixRoomMember(daveRtcMember);
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
const roomMembers = new Map(
[alice, bob, carol, dave].map((p) => [p.userId, p]),
);
export interface GridLayoutSummary {
type: "grid";
@@ -173,10 +183,23 @@ function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
function withCallViewModel(
remoteParticipants: Observable<RemoteParticipant[]>,
rtcMembers: Observable<Partial<CallMembership>[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const room = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => roomMembers.get(userId) ?? null,
});
const rtcSession = new MockRTCSession(
room,
localRtcMember,
[],
).withMemberships(rtcMembers);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants);
@@ -209,12 +232,7 @@ function withCallViewModel(
);
const vm = new CallViewModel(
mockMatrixRoom({
client: {
getUserId: () => "@carol:example.org",
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => members.get(userId) ?? null,
}),
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
@@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => {
a: [aliceParticipant, bobParticipant],
b: [],
}),
of([aliceRtcMember, bobRtcMember]),
hot(connectionInputMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
@@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => {
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
const modeInputMarbles = " a";
// First Bob speaks, then Dave, then Alice
const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms ";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
@@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => {
);
});
});
test("participants must have a MatrixRTCSession to be visible", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through a number of combinations of participants and MatrixRTC memberships
// Bob never has an MatrixRTC membership
const scenarioInputMarbles = " abcdec";
// Bob should never be visible
const expectedLayoutMarbles = "a-bc-b";
withCallViewModel(
hot(scenarioInputMarbles, {
a: [],
b: [bobParticipant],
c: [aliceParticipant, bobParticipant],
d: [aliceParticipant, daveParticipant, bobParticipant],
e: [aliceParticipant, daveParticipant, bobSharingScreen],
}),
hot(scenarioInputMarbles, {
a: [],
b: [],
c: [aliceRtcMember],
d: [aliceRtcMember, daveRtcMember],
e: [aliceRtcMember, daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
},
);
},
);
});
});
it("should show at least one tile per MatrixRTCSession", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through some combinations of MatrixRTC memberships
const scenarioInputMarbles = " abcd";
// There should always be one tile for each MatrixRTCSession
const expectedLayoutMarbles = "abcd";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [aliceRtcMember],
c: [aliceRtcMember, daveRtcMember],
d: [daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
d: {
type: "one-on-one",
local: "local:0",
remote: `${daveId}:0`,
},
},
);
},
);
});
});