studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'toger5/waitForNotificationAnswer' into toger5/call-pickup-state-decline-event

Browse Source
This commit is contained in:
Robin
2025-09-03 17:59:16 +02:00
parent 7724cbf9be 1e8b87408c
commit e9c43856d0
8 changed files with 266 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/url-params.md
View File

@@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
| `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. | | `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. |
| `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. | | `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. |
| `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. | | `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. |
| `autoLeaveWhenOthersLeft` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the app should automatically leave the call when there is no one left in the call. |
| `waitForCallPickup` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires. |
### Widget-only parameters ### Widget-only parameters

10
src/UrlParams.ts
View File

@@ -226,7 +226,7 @@ export interface UrlConfiguration {
* - play a sound that indicates that it is awaiting an answer * - play a sound that indicates that it is awaiting an answer
* - auto-dismiss the call widget once the notification lifetime expires on the receivers side. * - auto-dismiss the call widget once the notification lifetime expires on the receivers side.
*/ */
shouldWaitForCallPickup: boolean; waitForCallPickup: boolean;
} }
// If you need to add a new flag to this interface, prefer a name that describes // If you need to add a new flag to this interface, prefer a name that describes
@@ -358,7 +358,7 @@ export const getUrlParams = (
returnToLobby: false, returnToLobby: false,
sendNotificationType: "notification" as RTCNotificationType, sendNotificationType: "notification" as RTCNotificationType,
autoLeaveWhenOthersLeft: false, autoLeaveWhenOthersLeft: false,
shouldWaitForCallPickup: false, waitForCallPickup: false,
}; };
switch (intent) { switch (intent) {
case UserIntent.StartNewCall: case UserIntent.StartNewCall:
@@ -378,7 +378,7 @@ export const getUrlParams = (
...inAppDefault, ...inAppDefault,
skipLobby: true, skipLobby: true,
autoLeaveWhenOthersLeft: true, autoLeaveWhenOthersLeft: true,
shouldWaitForCallPickup: true, waitForCallPickup: true,
}; };
break; break;
case UserIntent.JoinExistingCallDM: case UserIntent.JoinExistingCallDM:
@@ -404,7 +404,7 @@ export const getUrlParams = (
returnToLobby: false, returnToLobby: false,
sendNotificationType: undefined, sendNotificationType: undefined,
autoLeaveWhenOthersLeft: false, autoLeaveWhenOthersLeft: false,
shouldWaitForCallPickup: false, waitForCallPickup: false,
}; };
} }
@@ -456,7 +456,7 @@ export const getUrlParams = (
"ring", "ring",
"notification", "notification",
]), ]),
shouldWaitForCallPickup: parser.getFlag("shouldWaitForCallPickup"), waitForCallPickup: parser.getFlag("waitForCallPickup"),
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
}; };

10
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -19,10 +19,7 @@ import { act } from "react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { mockRtcMembership } from "../utils/test"; import { mockRtcMembership } from "../utils/test";
import { import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext"; import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils"; import { prefetchSounds } from "../soundUtils";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
@@ -33,6 +30,7 @@ import {
local, local,
localRtcMember, localRtcMember,
} from "../utils/test-fixtures"; } from "../utils/test-fixtures";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel";
vitest.mock("../useAudioContext"); vitest.mock("../useAudioContext");
vitest.mock("../soundUtils"); vitest.mock("../soundUtils");
@@ -172,7 +170,7 @@ test("should not play a sound when a hand raise is retracted", () => {
}, },
}); });
}); });
expect(playSound).toHaveBeenCalledTimes(2); expect(playSound).toHaveBeenCalledTimes(1);
act(() => { act(() => {
handRaisedSubject$.next({ handRaisedSubject$.next({
["foo"]: { ["foo"]: {
@@ -182,5 +180,5 @@ test("should not play a sound when a hand raise is retracted", () => {
}, },
}); });
}); });
expect(playSound).toHaveBeenCalledTimes(2); expect(playSound).toHaveBeenCalledTimes(1);
}); });

49
src/room/CallEventAudioRenderer.tsx
View File

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/ */
import { type ReactNode, useEffect } from "react"; import { type ReactNode, useEffect } from "react";
import { filter, interval, throttle } from "rxjs";
import { type CallViewModel } from "../state/CallViewModel"; import { type CallViewModel } from "../state/CallViewModel";
import joinCallSoundMp3 from "../sound/join_call.mp3"; import joinCallSoundMp3 from "../sound/join_call.mp3";
@@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils"; import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest"; import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
export const callEventAudioSounds = prefetchSounds({ export const callEventAudioSounds = prefetchSounds({
join: { join: {
mp3: joinCallSoundMp3, mp3: joinCallSoundMp3,
@@ -60,37 +54,18 @@ export function CallEventAudioRenderer({
const audioEngineRef = useLatest(audioEngineCtx); const audioEngineRef = useLatest(audioEngineCtx);
useEffect(() => { useEffect(() => {
const joinSub = vm.participantChanges$ const joinSub = vm.joinSoundEffect$.subscribe(
.pipe( () => void audioEngineRef.current?.playSound("join"),
filter( );
({ joined, ids }) => const leftSub = vm.leaveSoundEffect$.subscribe(
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0, () => void audioEngineRef.current?.playSound("left"),
), );
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), const handRaisedSub = vm.newHandRaised$.subscribe(
) () => void audioEngineRef.current?.playSound("raiseHand"),
.subscribe(() => { );
void audioEngineRef.current?.playSound("join"); const screenshareSub = vm.newScreenShare$.subscribe(
}); () => void audioEngineRef.current?.playSound("screenshareStarted"),
);
const leftSub = vm.participantChanges$
.pipe(
filter(
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
void audioEngineRef.current?.playSound("left");
});
const handRaisedSub = vm.newHandRaised$.subscribe(() => {
void audioEngineRef.current?.playSound("raiseHand");
});
const screenshareSub = vm.newScreenShare$.subscribe(() => {
void audioEngineRef.current?.playSound("screenshareStarted");
});
return (): void => { return (): void => {
joinSub.unsubscribe(); joinSub.unsubscribe();

2
src/room/InCallView.tsx
View File

@@ -321,7 +321,7 @@ export const InCallView: FC<InCallViewProps> = ({
const showFooter = useBehavior(vm.showFooter$); const showFooter = useBehavior(vm.showFooter$);
const earpieceMode = useBehavior(vm.earpieceMode$); const earpieceMode = useBehavior(vm.earpieceMode$);
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave); useSubscription(vm.autoLeave$, onLeave);
// Ideally we could detect taps by listening for click events and checking // Ideally we could detect taps by listening for click events and checking
// that the pointerType of the event is "touch", but this isn't yet supported // that the pointerType of the event is "touch", but this isn't yet supported

259
src/state/CallViewModel.test.ts
View File

@@ -25,6 +25,7 @@ import {
RoomEvent as MatrixRoomEvent, RoomEvent as MatrixRoomEvent,
MatrixEvent, MatrixEvent,
type IRoomTimelineData, type IRoomTimelineData,
EventType,
} from "matrix-js-sdk"; } from "matrix-js-sdk";
import { import {
ConnectionState, ConnectionState,
@@ -318,7 +319,7 @@ function withCallViewModel(
const roomEventSelectorSpy = vi const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector") .spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of()); .mockImplementation((_room, _eventType) => of());
const livekitRoom = mockLivekitRoom( const livekitRoom = mockLivekitRoom(
{ localParticipant }, { localParticipant },
@@ -1078,9 +1079,9 @@ it("should rank raised hands above video feeds and below speakers and presenters
}); });
function nooneEverThere$<T>( function nooneEverThere$<T>(
hot: (marbles: string, values: Record<string, T[]>) => Observable<T[]>, behavior: (marbles: string, values: Record<string, T[]>) => Behavior<T[]>,
): Observable<T[]> { ): Behavior<T[]> {
return hot("a-b-c-d", { return behavior("a-b-c-d", {
a: [], // Start empty a: [], // Start empty
b: [], // Alice joins b: [], // Alice joins
c: [], // Alice still there c: [], // Alice still there
@@ -1089,12 +1090,12 @@ function nooneEverThere$<T>(
} }
function participantJoinLeave$( function participantJoinLeave$(
hot: ( behavior: (
marbles: string, marbles: string,
values: Record<string, RemoteParticipant[]>, values: Record<string, RemoteParticipant[]>,
) => Observable<RemoteParticipant[]>, ) => Behavior<RemoteParticipant[]>,
): Observable<RemoteParticipant[]> { ): Behavior<RemoteParticipant[]> {
return hot("a-b-c-d", { return behavior("a-b-c-d", {
a: [], // Start empty a: [], // Start empty
b: [aliceParticipant], // Alice joins b: [aliceParticipant], // Alice joins
c: [aliceParticipant], // Alice still there c: [aliceParticipant], // Alice still there
@@ -1103,12 +1104,12 @@ function participantJoinLeave$(
} }
function rtcMemberJoinLeave$( function rtcMemberJoinLeave$(
hot: ( behavior: (
marbles: string, marbles: string,
values: Record<string, CallMembership[]>, values: Record<string, CallMembership[]>,
) => Observable<CallMembership[]>, ) => Behavior<CallMembership[]>,
): Observable<CallMembership[]> { ): Behavior<CallMembership[]> {
return hot("a-b-c-d", { return behavior("a-b-c-d", {
a: [localRtcMember], // Start empty a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember], // Alice still there c: [localRtcMember, aliceRtcMember], // Alice still there
@@ -1116,47 +1117,15 @@ function rtcMemberJoinLeave$(
}); });
} }
test("allOthersLeft$ emits only when someone joined and then all others left", () => { test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ hot, expectObservable, scope }) => { withTestScheduler(({ behavior, expectObservable }) => {
// Test scenario 1: No one ever joins - should only emit initial false and never emit again
withCallViewModel(
{ remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) },
(vm) => {
expectObservable(vm.allOthersLeft$).toBe("n------", { n: false });
},
);
});
});
test("allOthersLeft$ emits true when someone joined and then all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), rtcMembers$: rtcMemberJoinLeave$(behavior),
}, },
(vm) => { (vm) => {
expectObservable(vm.allOthersLeft$).toBe( expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined });
"n-----u", // false initially, then at frame 6: true then false emissions in same frame
{ n: false, u: true }, // map(() => {})
);
},
);
});
});
test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe(
"------e", // false initially, then at frame 6: true then false emissions in same frame
{ e: undefined },
);
}, },
{ {
autoLeaveWhenOthersLeft: true, autoLeaveWhenOthersLeft: true,
@@ -1166,15 +1135,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is
}); });
}); });
test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
withTestScheduler(({ hot, expectObservable, scope }) => { withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: scope.behavior(nooneEverThere$(hot), []), remoteParticipants$: nooneEverThere$(behavior),
rtcMembers$: scope.behavior(nooneEverThere$(hot), []), rtcMembers$: nooneEverThere$(behavior),
}, },
(vm) => { (vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); expectObservable(vm.autoLeave$).toBe("-");
}, },
{ {
autoLeaveWhenOthersLeft: true, autoLeaveWhenOthersLeft: true,
@@ -1184,15 +1153,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena
}); });
}); });
test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => { withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), rtcMembers$: rtcMemberJoinLeave$(behavior),
}, },
(vm) => { (vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); expectObservable(vm.autoLeave$).toBe("-");
}, },
{ {
autoLeaveWhenOthersLeft: false, autoLeaveWhenOthersLeft: false,
@@ -1202,31 +1171,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option
}); });
}); });
test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => { withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: scope.behavior( remoteParticipants$: behavior("a-b-c-d", {
hot("a-b-c-d", { a: [], // Alone
a: [], // Alone b: [aliceParticipant], // Alice joins
b: [aliceParticipant], // Alice joins c: [aliceParticipant],
c: [aliceParticipant], d: [], // Local joins with a second device
d: [], // Local joins with a second device }),
}), rtcMembers$: behavior("a-b-c-d", {
[], //Alice leaves a: [localRtcMember], // Start empty
), b: [localRtcMember, aliceRtcMember], // Alice joins
rtcMembers$: scope.behavior( c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
hot("a-b-c-d", { d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
a: [localRtcMember], // Start empty }),
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
}),
[],
),
}, },
(vm) => { (vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", { expectObservable(vm.autoLeave$).toBe("------(e|)", {
e: undefined, e: undefined,
}); });
}, },
@@ -1238,18 +1201,12 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option
}); });
}); });
describe("shouldWaitForCallPickup$", () => { describe("waitForCallPickup$", () => {
test("unknown -> ringing -> timeout when notified and nobody joins", () => { test("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => { withTestScheduler(({ schedule, expectObservable }) => {
// No one ever joins (only local user) // No one ever joins (only local user)
withCallViewModel( withCallViewModel(
{ { remoteParticipants$: constant([]) },
remoteParticipants$: scope.behavior(hot("a", { a: [] }), []),
rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []),
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
},
(vm, rtcSession) => { (vm, rtcSession) => {
// Fire a call notification at 10ms with lifetime 30ms // Fire a call notification at 10ms with lifetime 30ms
schedule(" 10ms r", { schedule(" 10ms r", {
@@ -1273,7 +1230,7 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1281,25 +1238,18 @@ describe("shouldWaitForCallPickup$", () => {
}); });
test("ringing -> success if someone joins before timeout", () => { test("ringing -> success if someone joins before timeout", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => { withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) // Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const remote$ = scope.behavior(
hot("a--b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: remote$, remoteParticipants$: behavior("a 19ms b", {
rtcMembers$: rtc$, a: [],
connectionState$: of(ConnectionState.Connected), b: [aliceParticipant],
}),
rtcMembers$: behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
}, },
(vm, rtcSession) => { (vm, rtcSession) => {
// Notify at 5ms so we enter ringing, then success at 20ms // Notify at 5ms so we enter ringing, then success at 20ms
@@ -1320,13 +1270,14 @@ describe("shouldWaitForCallPickup$", () => {
}, },
}); });
expectObservable(vm.callPickupState$).toBe("a 2ms c", { expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown", a: "unknown",
b: "ringing",
c: "success", c: "success",
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1334,27 +1285,18 @@ describe("shouldWaitForCallPickup$", () => {
}); });
test("success when someone joins before we notify", () => { test("success when someone joins before we notify", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => { withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Join at 10ms, notify later at 20ms (state should stay success) // Join at 10ms, notify later at 20ms (state should stay success)
const remote$ = scope.behavior(
hot("a-b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a-b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: remote$, remoteParticipants$: behavior("a 9ms b", {
rtcMembers$: rtc$, a: [],
connectionState$: of(ConnectionState.Connected), b: [aliceParticipant],
speaking: new Map(), }),
mediaDevices: mockMediaDevices({}), rtcMembers$: behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
}, },
(vm, rtcSession) => { (vm, rtcSession) => {
schedule(" 20ms r", { schedule(" 20ms r", {
@@ -1370,13 +1312,13 @@ describe("shouldWaitForCallPickup$", () => {
); );
}, },
}); });
expectObservable(vm.callPickupState$).toBe("a 1ms b", { expectObservable(vm.callPickupState$).toBe("a 9ms b", {
a: "unknown", a: "unknown",
b: "success", b: "success",
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1384,21 +1326,15 @@ describe("shouldWaitForCallPickup$", () => {
}); });
test("notify without lifetime -> immediate timeout", () => { test("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => { withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel( withCallViewModel(
{ {},
remoteParticipants$: scope.behavior(hot("a", { a: [] }), []),
rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []),
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
},
(vm, rtcSession) => { (vm, rtcSession) => {
schedule(" 10ms r", { schedule(" 10ms r", {
r: () => { r: () => {
rtcSession.emit( rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification, MatrixRTCSessionEvent.DidSendCallNotification,
{ event_id: "$notif4", lifetime: 0 } as unknown as { { event_id: "$notif4" } as unknown as {
event_id: string; event_id: string;
} & IRTCNotificationContent, // no lifetime } & IRTCNotificationContent, // no lifetime
{ event_id: "$notif4" } as unknown as { { event_id: "$notif4" } as unknown as {
@@ -1413,34 +1349,25 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
}); });
}); });
test("stays null when shouldWaitForCallPickup=false", () => { test("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => { withTestScheduler(({ behavior, schedule, expectObservable }) => {
const remote$ = scope.behavior(
hot("a--b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
withCallViewModel( withCallViewModel(
{ {
remoteParticipants$: remote$, remoteParticipants$: behavior("a--b", {
rtcMembers$: rtc$, a: [],
connectionState$: of(ConnectionState.Connected), b: [aliceParticipant],
speaking: new Map(), }),
mediaDevices: mockMediaDevices({}), rtcMembers$: behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
}, },
(vm, rtcSession) => { (vm, rtcSession) => {
schedule(" 5ms r", { schedule(" 5ms r", {
@@ -1461,7 +1388,7 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: false, waitForCallPickup: false,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1494,7 +1421,15 @@ describe("shouldWaitForCallPickup$", () => {
// Emit decline timeline event with id matching the notification // Emit decline timeline event with id matching the notification
rtcSession.room.emit( rtcSession.room.emit(
MatrixRoomEvent.Timeline, MatrixRoomEvent.Timeline,
new MatrixEvent({ event_id: "$decl1", type: "m.rtc.decline" }), new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl1",
},
},
}),
rtcSession.room, rtcSession.room,
undefined, undefined,
false, false,
@@ -1509,7 +1444,7 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1557,7 +1492,7 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );
@@ -1604,7 +1539,7 @@ describe("shouldWaitForCallPickup$", () => {
}); });
}, },
{ {
shouldWaitForCallPickup: true, waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
}, },
); );

343
src/state/CallViewModel.ts
View File

@@ -30,18 +30,22 @@ import {
import { import {
BehaviorSubject, BehaviorSubject,
EMPTY, EMPTY,
NEVER,
type Observable, type Observable,
Subject, Subject,
combineLatest, combineLatest,
concat, concat,
distinctUntilChanged, distinctUntilChanged,
endWith,
filter, filter,
forkJoin, forkJoin,
fromEvent, fromEvent,
ignoreElements,
map, map,
merge, merge,
mergeMap, mergeMap,
of, of,
pairwise,
race, race,
scan, scan,
skip, skip,
@@ -50,6 +54,8 @@ import {
switchMap, switchMap,
switchScan, switchScan,
take, take,
takeUntil,
throttleTime,
timer, timer,
withLatestFrom, withLatestFrom,
} from "rxjs"; } from "rxjs";
@@ -109,7 +115,7 @@ import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array"; import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices"; import { type MediaDevices } from "./MediaDevices";
import { type Behavior } from "./Behavior"; import { constant, type Behavior } from "./Behavior";
export interface CallViewModelOptions { export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem; encryptionSystem: EncryptionSystem;
@@ -118,13 +124,18 @@ export interface CallViewModelOptions {
* If the call is started in a way where we want it to behave like a telephone usecase * If the call is started in a way where we want it to behave like a telephone usecase
* If we sent a notification event, we want the ui to show a ringing state * If we sent a notification event, we want the ui to show a ringing state
*/ */
shouldWaitForCallPickup?: boolean; waitForCallPickup?: boolean;
} }
// How long we wait after a focus switch before showing the real participant // How long we wait after a focus switch before showing the real participant
// list again // list again
const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
// This is the number of participants that we think constitutes a "small" call // This is the number of participants that we think constitutes a "small" call
// on mobile. No spotlight tile should be shown below this threshold. // on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3; const smallMobileCallThreshold = 3;
@@ -437,14 +448,7 @@ export class CallViewModel extends ViewModel {
MembershipManagerEvent.StatusChanged, MembershipManagerEvent.StatusChanged,
).pipe( ).pipe(
startWith(null), startWith(null),
map( map(() => this.matrixRTCSession.membershipStatus === Status.Connected),
() =>
(
this.matrixRTCSession as unknown as {
membershipStatus?: Status;
}
).membershipStatus === Status.Connected,
),
), ),
// Also watch out for warnings that we've likely hit a timeout and our // Also watch out for warnings that we've likely hit a timeout and our
// delayed leave event is being sent (this condition is here because it // delayed leave event is being sent (this condition is here because it
@@ -455,11 +459,7 @@ export class CallViewModel extends ViewModel {
MembershipManagerEvent.ProbablyLeft, MembershipManagerEvent.ProbablyLeft,
).pipe( ).pipe(
startWith(null), startWith(null),
map( map(() => this.matrixRTCSession.probablyLeft !== true),
() =>
(this.matrixRTCSession as unknown as { probablyLeft?: boolean })
.probablyLeft !== true,
),
), ),
), ),
); );
@@ -576,6 +576,17 @@ export class CallViewModel extends ViewModel {
) )
.pipe(pauseWhen(this.pretendToBeDisconnected$)); .pipe(pauseWhen(this.pretendToBeDisconnected$));
private readonly memberships$ = this.scope.behavior(
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(
startWith(null),
pauseWhen(this.pretendToBeDisconnected$),
map(() => this.matrixRTCSession.memberships),
),
);
/** /**
* Displaynames for each member of the call. This will disambiguate * Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members * any displaynames that clashes with another member. Only members
@@ -585,18 +596,17 @@ export class CallViewModel extends ViewModel {
// than on Chrome/Firefox). This means it is important that we multicast the result so that we // than on Chrome/Firefox). This means it is important that we multicast the result so that we
// don't do this work more times than we need to. This is achieved by converting to a behavior: // don't do this work more times than we need to. This is achieved by converting to a behavior:
public readonly memberDisplaynames$ = this.scope.behavior( public readonly memberDisplaynames$ = this.scope.behavior(
merge( // React to call memberships and also display name updates
// Handle call membership changes. // (calculateDisplayName implicitly depends on the room member data)
fromEvent( combineLatest(
this.matrixRTCSession, [
MatrixRTCSessionEvent.MembershipsChanged, this.memberships$,
), fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe(
// Handle room membership changes (and displayname updates) startWith(null),
fromEvent(this.matrixRoom, RoomStateEvent.Members), pauseWhen(this.pretendToBeDisconnected$),
).pipe( ),
startWith(null), ],
map(() => { (memberships, _members) => {
const memberships = this.matrixRTCSession.memberships;
const displaynameMap = new Map<string, string>(); const displaynameMap = new Map<string, string>();
const room = this.matrixRoom; const room = this.matrixRoom;
@@ -618,8 +628,7 @@ export class CallViewModel extends ViewModel {
); );
} }
return displaynameMap; return displaynameMap;
}), },
pauseWhen(this.pretendToBeDisconnected$),
), ),
); );
@@ -649,13 +658,7 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants$, this.remoteParticipants$,
observeParticipantMedia(this.livekitRoom.localParticipant), observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value$, duplicateTiles.value$,
// Also react to changes in the MatrixRTC session list. this.memberships$,
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)),
showNonMemberTiles.value$, showNonMemberTiles.value$,
]).pipe( ]).pipe(
scan( scan(
@@ -665,7 +668,7 @@ export class CallViewModel extends ViewModel {
remoteParticipants, remoteParticipants,
{ participant: localParticipant }, { participant: localParticipant },
duplicateTiles, duplicateTiles,
_membershipsChanged, memberships,
showNonMemberTiles, showNonMemberTiles,
], ],
) => { ) => {
@@ -673,7 +676,7 @@ export class CallViewModel extends ViewModel {
function* (this: CallViewModel): Iterable<[string, MediaItem]> { function* (this: CallViewModel): Iterable<[string, MediaItem]> {
const room = this.matrixRoom; const room = this.matrixRoom;
// m.rtc.members are the basis for calculating what is visible in the call // m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) { for (const rtcMember of memberships) {
const { member, id: livekitParticipantId } = const { member, id: livekitParticipantId } =
getRoomMemberFromRtcMember(rtcMember, room); getRoomMemberFromRtcMember(rtcMember, room);
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
@@ -839,205 +842,143 @@ export class CallViewModel extends ViewModel {
), ),
); );
/** public readonly joinSoundEffect$ = this.userMedia$.pipe(
* This observable tracks the currently connected participants. pairwise(),
* filter(
* - Each participant has one livekit connection ([prev, current]) =>
* - Each participant has a corresponding MatrixRTC membership state event current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
* - There can be multiple participants for one matrix user. current.length > prev.length,
*/
public readonly participantChanges$ = this.scope.behavior(
this.userMedia$.pipe(
map((mediaItems) => mediaItems.map((m) => m.id)),
scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
(prev, ids) => {
const left = prev.ids.filter((id) => !ids.includes(id));
const joined = ids.filter((id) => !prev.ids.includes(id));
return { ids, joined, left };
},
{ ids: [], joined: [], left: [] },
),
), ),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
public readonly leaveSoundEffect$ = this.userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
current.length < prev.length,
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
); );
/** /**
* The number of participants currently in the call. * The number of participants currently in the call.
* *
* - Each participant has one livekit connection
* - Each participant has a corresponding MatrixRTC membership state event * - Each participant has a corresponding MatrixRTC membership state event
* - There can be multiple participants for one matrix user. * - There can be multiple participants for one Matrix user if they join from
* multiple devices.
*/ */
public readonly participantCount$ = this.scope.behavior( public readonly participantCount$ = this.scope.behavior(
this.participantChanges$.pipe(map(({ ids }) => ids.length)), this.memberships$.pipe(map((ms) => ms.length)),
); );
/** private readonly allOthersLeft$ = this.memberships$.pipe(
* This observable tracks the matrix users that are currently in the call. pairwise(),
* There can be just one matrix user with multiple participants (see also participantChanges$) filter(
*/ ([prev, current]) =>
public readonly matrixUserChanges$ = this.scope.behavior( current.every((m) => m.sender === this.userId) &&
this.userMedia$.pipe( prev.some((m) => m.sender !== this.userId),
map(
(mediaItems) =>
new Set(
mediaItems
.map((m) => m.vm.member?.userId)
.filter((id) => id !== undefined),
),
),
scan<
Set<string>,
{
userIds: Set<string>;
joinedUserIds: Set<string>;
leftUserIds: Set<string>;
}
>(
(prevState, userIds) => {
const left = new Set(
[...prevState.userIds].filter((id) => !userIds.has(id)),
);
const joined = new Set(
[...userIds].filter((id) => !prevState.userIds.has(id)),
);
return { userIds: userIds, joinedUserIds: joined, leftUserIds: left };
},
{
userIds: new Set(),
joinedUserIds: new Set(),
leftUserIds: new Set(),
},
),
), ),
);
public readonly allOthersLeft$ = this.matrixUserChanges$.pipe(
map(({ userIds, leftUserIds }) => {
if (!this.userId) {
logger.warn("Could not access user ID to compute allOthersLeft");
return false;
}
return (
userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0
);
}),
startWith(false),
distinctUntilChanged(),
);
public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe(
distinctUntilChanged(),
filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false),
map(() => {}), map(() => {}),
take(1),
); );
public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft
? this.allOthersLeft$
: NEVER;
/** /**
* "unknown": We don't know if the RTC session decides to send a notify event yet. * Whenever the RTC session tells us that it intends to ring the remote
* It will only be known once we sent our own membership and know we were the first one to join. * participant's devices, this emits an Observable tracking the current state of
* "ringing": The notification event was sent. * that ringing process.
* "ringEnded": The notification events lifetime has timed out -> ringing stopped on all receiving clients.
*/ */
private readonly rtcNotificationEventState$: Observable< private readonly ring$: Observable<
{ state: "unknown" | "ringEnded" } | { state: "ringing"; event_id: string } Observable<"ringing" | "timeout" | "decline">
> = fromEvent< > = (
Parameters< fromEvent(
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] this.matrixRTCSession,
MatrixRTCSessionEvent.DidSendCallNotification,
) as Observable<
Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>
> >
>(this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification).pipe( ).pipe(
switchMap(([notificationEvent]) => { map(([notificationEvent]) => {
// event.lifetime is expected to be in ms // event.lifetime is expected to be in ms
const lifetime = notificationEvent?.lifetime ?? 0; const lifetime = notificationEvent?.lifetime ?? 0;
if (lifetime > 0) { return concat(
// Emit true immediately, then false after lifetime ms lifetime === 0
return concat( ? // If no lifetime, skip the ring state
of({ EMPTY
state: "ringing", : // Ring until lifetime ms have passed
event_id: notificationEvent.event_id, timer(lifetime).pipe(
} as { ignoreElements(),
state: "ringing"; startWith("ringing" as const),
event_id: string; ),
}), // The notification lifetime has timed out, meaning ringing has likely
timer(lifetime).pipe( // stopped on all receiving clients.
map(() => ({ state: "ringEnded" }) as { state: "ringEnded" }), of("timeout" as const),
NEVER,
).pipe(
takeUntil(
(
fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>
).pipe(
filter(
([event]) =>
event.getType() === EventType.RTCDecline &&
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id === notificationEvent.event_id,
),
), ),
); ),
} endWith("decline" as const),
// If no lifetime, the notify event is basically invalid and we enter ringEnded immediately. );
return of({ state: "ringEnded" } as { state: "ringEnded" });
}), }),
startWith({ state: "unknown" } as { state: "unknown" }),
); );
/** /**
* If some other matrix user has joined the call. It can start with true if there are already multiple matrix users. * Whether some Matrix user other than ourself is joined to the call.
*/ */
private readonly someoneElseJoined$ = this.matrixUserChanges$.pipe( private readonly someoneElseJoined$ = this.memberships$.pipe(
scan( map((ms) => ms.some((m) => m.sender !== this.userId)),
(someoneJoined, { joinedUserIds }) =>
someoneJoined || [...joinedUserIds].some((id) => id !== this.userId),
false,
),
startWith(this.matrixUserChanges$.value.userIds.size > 1),
); );
/** /**
* The current call pickup state of the call. * The current call pickup state of the call.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not. * Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices. * - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call. * The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. Stop audiovisual feedback. * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state. * - null: EC is configured to never show any waiting for answer state.
*/ */
public readonly callPickupState$: Behavior< public readonly callPickupState$ = this.options.waitForCallPickup
"unknown" | "ringing" | "timeout" | "success" | "decline" | null ? this.scope.behavior<
> = this.scope.behavior( "unknown" | "ringing" | "timeout" | "decline" | "success"
combineLatest([ >(
this.rtcNotificationEventState$, this.someoneElseJoined$.pipe(
this.someoneElseJoined$, switchMap((someoneElseJoined) =>
fromEvent<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>>( someoneElseJoined
this.matrixRoom, ? of("success" as const)
RoomEvent.Timeline, : // Show the ringing state of the most recent ringing attempt.
).pipe( this.ring$.pipe(switchAll()),
map(([event]) => { ),
if (event.getType() === EventType.RTCDecline) return event; // The state starts as 'unknown' because we don't know if the RTC
else return null; // session will actually send a notify event yet. It will only be
}), // known once we send our own membership and see that we were the
startWith(null), // first one to join.
), startWith("unknown" as const),
]).pipe( ),
map(([notificationEventState, someoneJoined, declineEvent]) => { )
// Never enter waiting for answer state if the app is not configured with waitingForAnswer. : constant(null);
if (!this.options.shouldWaitForCallPickup) return null;
// As soon as someone joins, we can consider the call "wait for answer" successful
if (someoneJoined) return "success";
switch (notificationEventState?.state) {
case "unknown":
return "unknown";
case "ringing":
// Check if the decline event corresponds to the current notification event
if (declineEvent?.getId() === notificationEventState.event_id) {
return "decline";
}
return "ringing";
case "ringEnded":
return "timeout";
default:
return "timeout";
}
}),
// Once we reach a terminal state, keep it
scan((prev, next) => {
if (prev === "decline" || prev === "timeout" || prev === "success") {
return prev;
}
return next;
}),
distinctUntilChanged(),
),
);
/** /**
* List of MediaItems that we want to display, that are of type ScreenShare * List of MediaItems that we want to display, that are of type ScreenShare

6
yarn.lock
View File

@@ -10046,8 +10046,8 @@ __metadata:
linkType: hard linkType: hard
"livekit-client@npm:^2.13.0": "livekit-client@npm:^2.13.0":
version: 2.15.5 version: 2.15.6
resolution: "livekit-client@npm:2.15.5" resolution: "livekit-client@npm:2.15.6"
dependencies: dependencies:
"@livekit/mutex": "npm:1.1.1" "@livekit/mutex": "npm:1.1.1"
"@livekit/protocol": "npm:1.39.3" "@livekit/protocol": "npm:1.39.3"
@@ -10060,7 +10060,7 @@ __metadata:
webrtc-adapter: "npm:^9.0.1" webrtc-adapter: "npm:^9.0.1"
peerDependencies: peerDependencies:
"@types/dom-mediacapture-record": ^1 "@types/dom-mediacapture-record": ^1
checksum: 10c0/52a70bdd39d802737ed7c25ae5d06daf9921156c4fc74f918009e86204430b2d200b66c55cefab949be4e5411cbc4d25eac92976f62f96b7226057a5b0706baa checksum: 10c0/f1ab6cdf2b85647036e9de906734c1394dac497da0bd879a29d0c587c437ada262021478fcef24df99b3489a39d97fe67ab33de0785ed0a63335da2fef577192
languageName: node languageName: node
linkType: hard linkType: hard