studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3466 from element-hq/toger5/waitForNotificationAnswer

Browse Source 
Add dialing/ringing state to CallViewModel (`callPickupState$`)
This commit is contained in:
Robin
2025-09-05 14:11:37 +02:00
committed by GitHub
parent 075a4cbe0d c599d6eae9
commit 7961bb35be
12 changed files with 456 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/url-params.md
View File

@@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
| `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. |
| `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. |
| `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. |
| `autoLeaveWhenOthersLeft` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the app should automatically leave the call when there is no one left in the call. |
| `waitForCallPickup` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires. |
### Widget-only parameters

15
src/UrlParams.ts
View File

@@ -216,6 +216,17 @@ export interface UrlConfiguration {
* This is one part to make the call matrixRTC session behave like a telephone call.
*/
autoLeaveWhenOthersLeft: boolean;
/**
* If the client should behave like it is awaiting an answer if a notification was sent (wait for call pick up).
* This is a no-op if not combined with sendNotificationType.
*
* This entails:
* - show ui that it is awaiting an answer
* - play a sound that indicates that it is awaiting an answer
* - auto-dismiss the call widget once the notification lifetime expires on the receivers side.
*/
waitForCallPickup: boolean;
}
// If you need to add a new flag to this interface, prefer a name that describes
@@ -347,6 +358,7 @@ export const getUrlParams = (
returnToLobby: false,
sendNotificationType: "notification" as RTCNotificationType,
autoLeaveWhenOthersLeft: false,
waitForCallPickup: false,
};
switch (intent) {
case UserIntent.StartNewCall:
@@ -366,6 +378,7 @@ export const getUrlParams = (
...inAppDefault,
skipLobby: true,
autoLeaveWhenOthersLeft: true,
waitForCallPickup: true,
};
break;
case UserIntent.JoinExistingCallDM:
@@ -391,6 +404,7 @@ export const getUrlParams = (
returnToLobby: false,
sendNotificationType: undefined,
autoLeaveWhenOthersLeft: false,
waitForCallPickup: false,
};
}
@@ -442,6 +456,7 @@ export const getUrlParams = (
"ring",
"notification",
]),
waitForCallPickup: parser.getFlag("waitForCallPickup"),
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
};

4
src/livekit/useECConnectionState.test.tsx
View File

@@ -94,7 +94,7 @@ describe("Leaking connection prevention", () => {
test("Should cancel pending connections when the component is unmounted", async () => {
const connectCall = vi.fn();
const pendingConnection = Promise.withResolvers<void>();
// let pendingDisconnection = defer<void>()
// let pendingDisconnection = Promise.withResolvers<void>()
const disconnectMock = vi.fn();
const mockRoom = {
@@ -142,7 +142,7 @@ describe("Leaking connection prevention", () => {
test("Should cancel about to open but not yet opened connection", async () => {
const createTracksCall = vi.fn();
const pendingCreateTrack = Promise.withResolvers<void>();
// let pendingDisconnection = defer<void>()
// let pendingDisconnection = Promise.withResolvers<void>()
const disconnectMock = vi.fn();
const connectMock = vi.fn();

11
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -19,10 +19,7 @@ import { act } from "react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { mockRtcMembership } from "../utils/test";
import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
@@ -33,6 +30,7 @@ import {
local,
localRtcMember,
} from "../utils/test-fixtures";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel";
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
@@ -158,6 +156,7 @@ test("should not play a sound when a hand raise is retracted", () => {
]);
render(<CallEventAudioRenderer vm={vm} />);
playSound.mockClear();
act(() => {
handRaisedSubject$.next({
["foo"]: {
@@ -172,7 +171,7 @@ test("should not play a sound when a hand raise is retracted", () => {
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand");
act(() => {
handRaisedSubject$.next({
["foo"]: {
@@ -182,5 +181,5 @@ test("should not play a sound when a hand raise is retracted", () => {
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand");
});

49
src/room/CallEventAudioRenderer.tsx
View File

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useEffect } from "react";
import { filter, interval, throttle } from "rxjs";
import { type CallViewModel } from "../state/CallViewModel";
import joinCallSoundMp3 from "../sound/join_call.mp3";
@@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
export const callEventAudioSounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
@@ -60,37 +54,18 @@ export function CallEventAudioRenderer({
const audioEngineRef = useLatest(audioEngineCtx);
useEffect(() => {
const joinSub = vm.participantChanges$
.pipe(
filter(
({ joined, ids }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
void audioEngineRef.current?.playSound("join");
});
const leftSub = vm.participantChanges$
.pipe(
filter(
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
void audioEngineRef.current?.playSound("left");
});
const handRaisedSub = vm.newHandRaised$.subscribe(() => {
void audioEngineRef.current?.playSound("raiseHand");
});
const screenshareSub = vm.newScreenShare$.subscribe(() => {
void audioEngineRef.current?.playSound("screenshareStarted");
});
const joinSub = vm.joinSoundEffect$.subscribe(
() => void audioEngineRef.current?.playSound("join"),
);
const leftSub = vm.leaveSoundEffect$.subscribe(
() => void audioEngineRef.current?.playSound("left"),
);
const handRaisedSub = vm.newHandRaised$.subscribe(
() => void audioEngineRef.current?.playSound("raiseHand"),
);
const screenshareSub = vm.newScreenShare$.subscribe(
() => void audioEngineRef.current?.playSound("screenshareStarted"),
);
return (): void => {
joinSub.unsubscribe();

1
src/room/GroupCallView.tsx
View File

@@ -453,7 +453,6 @@ export const GroupCallView: FC<Props> = ({
matrixInfo={matrixInfo}
rtcSession={rtcSession as MatrixRTCSession}
matrixRoom={room}
participantCount={participantCount}
onLeave={onLeave}
header={header}
muteStates={muteStates}

1
src/room/InCallView.test.tsx
View File

@@ -177,7 +177,6 @@ function createInCallView(): RenderResult & {
}}
matrixRoom={room}
livekitRoom={livekitRoom}
participantCount={0}
onLeave={function (): void {
throw new Error("Function not implemented.");
}}

5
src/room/InCallView.tsx
View File

@@ -216,7 +216,6 @@ export interface InCallViewProps {
matrixRoom: MatrixRoom;
livekitRoom: LivekitRoom;
muteStates: MuteStates;
participantCount: number;
/** Function to call when the user explicitly ends the call */
onLeave: () => void;
header: HeaderStyle;
@@ -233,7 +232,6 @@ export const InCallView: FC<InCallViewProps> = ({
matrixRoom,
livekitRoom,
muteStates,
participantCount,
onLeave,
header: headerStyle,
connState,
@@ -312,6 +310,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);
const layout = useBehavior(vm.layout$);
@@ -322,7 +321,7 @@ export const InCallView: FC<InCallViewProps> = ({
const showFooter = useBehavior(vm.showFooter$);
const earpieceMode = useBehavior(vm.earpieceMode$);
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave);
useSubscription(vm.autoLeave$, onLeave);
// Ideally we could detect taps by listening for click events and checking
// that the pointerType of the event is "touch", but this isn't yet supported

28
src/room/__snapshots__/InCallView.test.tsx.snap
View File

@@ -49,6 +49,34 @@ exports[`InCallView > rendering > renders 1`] = `
</svg>
</span>
</div>
<div
class="participantsLine"
>
<svg
aria-label="Participants"
fill="currentColor"
height="20"
viewBox="0 0 24 24"
width="20"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M9.175 13.825Q10.35 15 12 15t2.825-1.175T16 11t-1.175-2.825T12 7 9.175 8.175 8 11t1.175 2.825m4.237-1.412A1.93 1.93 0 0 1 12 13q-.825 0-1.412-.588A1.93 1.93 0 0 1 10 11q0-.825.588-1.412A1.93 1.93 0 0 1 12 9q.825 0 1.412.588Q14 10.175 14 11t-.588 1.412"
/>
<path
d="M22 12c0 5.523-4.477 10-10 10S2 17.523 2 12 6.477 2 12 2s10 4.477 10 10m-2 0a8 8 0 1 0-16 0 8 8 0 0 0 16 0"
/>
<path
d="M16.23 18.792a13 13 0 0 0-1.455-.455 11.6 11.6 0 0 0-5.55 0q-.73.18-1.455.455a8 8 0 0 1-1.729-1.454q1.336-.618 2.709-.95A13.8 13.8 0 0 1 12 16q1.65 0 3.25.387 1.373.333 2.709.95a8 8 0 0 1-1.73 1.455"
/>
</svg>
<span
class="_typography_6v6n8_153 _font-body-sm-medium_6v6n8_41"
data-testid="roomHeader_participants_count"
>
2
</span>
</div>
</div>
</div>
<div

317
src/state/CallViewModel.test.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished, it } from "vitest";
import { test, vi, onTestFinished, it, describe } from "vitest";
import EventEmitter from "events";
import {
BehaviorSubject,
@@ -32,6 +32,9 @@ import {
Status,
type CallMembership,
type MatrixRTCSession,
type IRTCNotificationContent,
type ICallNotifyContent,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { deepCompare } from "matrix-js-sdk/lib/utils";
@@ -308,7 +311,7 @@ function withCallViewModel(
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of());
.mockImplementation((_room, _eventType) => of());
const livekitRoom = mockLivekitRoom(
{ localParticipant },
@@ -1068,9 +1071,9 @@ it("should rank raised hands above video feeds and below speakers and presenters
});
function nooneEverThere$<T>(
hot: (marbles: string, values: Record<string, T[]>) => Observable<T[]>,
): Observable<T[]> {
return hot("a-b-c-d", {
behavior: (marbles: string, values: Record<string, T[]>) => Behavior<T[]>,
): Behavior<T[]> {
return behavior("a-b-c-d", {
a: [], // Start empty
b: [], // Alice joins
c: [], // Alice still there
@@ -1079,12 +1082,12 @@ function nooneEverThere$<T>(
}
function participantJoinLeave$(
hot: (
behavior: (
marbles: string,
values: Record<string, RemoteParticipant[]>,
) => Observable<RemoteParticipant[]>,
): Observable<RemoteParticipant[]> {
return hot("a-b-c-d", {
) => Behavior<RemoteParticipant[]>,
): Behavior<RemoteParticipant[]> {
return behavior("a-b-c-d", {
a: [], // Start empty
b: [aliceParticipant], // Alice joins
c: [aliceParticipant], // Alice still there
@@ -1093,12 +1096,12 @@ function participantJoinLeave$(
}
function rtcMemberJoinLeave$(
hot: (
behavior: (
marbles: string,
values: Record<string, CallMembership[]>,
) => Observable<CallMembership[]>,
): Observable<CallMembership[]> {
return hot("a-b-c-d", {
) => Behavior<CallMembership[]>,
): Behavior<CallMembership[]> {
return behavior("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember], // Alice still there
@@ -1106,47 +1109,15 @@ function rtcMemberJoinLeave$(
});
}
test("allOthersLeft$ emits only when someone joined and then all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
// Test scenario 1: No one ever joins - should only emit initial false and never emit again
withCallViewModel(
{ remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) },
(vm) => {
expectObservable(vm.allOthersLeft$).toBe("n------", { n: false });
},
);
});
});
test("allOthersLeft$ emits true when someone joined and then all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: rtcMemberJoinLeave$(behavior),
},
(vm) => {
expectObservable(vm.allOthersLeft$).toBe(
"n-----u", // false initially, then at frame 6: true then false emissions in same frame
{ n: false, u: true }, // map(() => {})
);
},
);
});
});
test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe(
"------e", // false initially, then at frame 6: true then false emissions in same frame
{ e: undefined },
);
expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined });
},
{
autoLeaveWhenOthersLeft: true,
@@ -1156,15 +1127,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is
});
});
test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(nooneEverThere$(hot), []),
rtcMembers$: scope.behavior(nooneEverThere$(hot), []),
remoteParticipants$: nooneEverThere$(behavior),
rtcMembers$: nooneEverThere$(behavior),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
expectObservable(vm.autoLeave$).toBe("-");
},
{
autoLeaveWhenOthersLeft: true,
@@ -1174,15 +1145,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena
});
});
test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: rtcMemberJoinLeave$(behavior),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
expectObservable(vm.autoLeave$).toBe("-");
},
{
autoLeaveWhenOthersLeft: false,
@@ -1192,31 +1163,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option
});
});
test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(
hot("a-b-c-d", {
a: [], // Alone
b: [aliceParticipant], // Alice joins
c: [aliceParticipant],
d: [], // Local joins with a second device
}),
[], //Alice leaves
),
rtcMembers$: scope.behavior(
hot("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
}),
[],
),
remoteParticipants$: behavior("a-b-c-d", {
a: [], // Alone
b: [aliceParticipant], // Alice joins
c: [aliceParticipant],
d: [], // Local joins with a second device
}),
rtcMembers$: behavior("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
}),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", {
expectObservable(vm.autoLeave$).toBe("------(e|)", {
e: undefined,
});
},
@@ -1228,6 +1193,196 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option
});
});
describe("waitForCallPickup$", () => {
test("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// No one ever joins (only local user)
withCallViewModel(
{ remoteParticipants$: constant([]) },
(vm, rtcSession) => {
// Fire a call notification at 10ms with lifetime 30ms
schedule(" 10ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{ lifetime: 30 } as unknown as {
event_id: string;
} & IRTCNotificationContent,
{} as unknown as { event_id: string } & ICallNotifyContent,
);
},
});
expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
test("ringing -> success if someone joins before timeout", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
withCallViewModel(
{
remoteParticipants$: behavior("a 19ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
// Notify at 5ms so we enter ringing, then success at 20ms
schedule(" 5ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{ lifetime: 100 } as unknown as {
event_id: string;
} & IRTCNotificationContent,
{} as unknown as {
event_id: string;
} & ICallNotifyContent,
);
},
});
expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown",
b: "ringing",
c: "success",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
test("success when someone joins before we notify", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Join at 10ms, notify later at 20ms (state should stay success)
withCallViewModel(
{
remoteParticipants$: behavior("a 9ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
schedule(" 20ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{ lifetime: 50 } as unknown as {
event_id: string;
} & IRTCNotificationContent,
{} as unknown as {
event_id: string;
} & ICallNotifyContent,
);
},
});
expectObservable(vm.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "success",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
test("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel(
{},
(vm, rtcSession) => {
schedule(" 10ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{} as unknown as {
event_id: string;
} & IRTCNotificationContent, // no lifetime
{} as unknown as {
event_id: string;
} & ICallNotifyContent,
);
},
});
expectObservable(vm.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "timeout",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
test("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: behavior("a--b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
schedule(" 5ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{ lifetime: 30 } as unknown as {
event_id: string;
} & IRTCNotificationContent,
{} as unknown as {
event_id: string;
} & ICallNotifyContent,
);
},
});
expectObservable(vm.callPickupState$).toBe("(n)", {
n: null,
});
},
{
waitForCallPickup: false,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
});
test("audio output changes when toggling earpiece mode", () => {
withTestScheduler(({ schedule, expectObservable }) => {
getUrlParams.mockReturnValue({ controlledAudioDevices: true });

245
src/state/CallViewModel.ts
View File

@@ -27,6 +27,7 @@ import {
import {
BehaviorSubject,
EMPTY,
NEVER,
type Observable,
Subject,
combineLatest,
@@ -35,10 +36,12 @@ import {
filter,
forkJoin,
fromEvent,
ignoreElements,
map,
merge,
mergeMap,
of,
pairwise,
race,
scan,
skip,
@@ -47,12 +50,15 @@ import {
switchMap,
switchScan,
take,
takeUntil,
throttleTime,
timer,
withLatestFrom,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type CallMembership,
type IRTCNotificationContent,
type MatrixRTCSession,
MatrixRTCSessionEvent,
MembershipManagerEvent,
@@ -105,16 +111,27 @@ import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices";
import { type Behavior } from "./Behavior";
import { constant, type Behavior } from "./Behavior";
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
autoLeaveWhenOthersLeft?: boolean;
/**
* If the call is started in a way where we want it to behave like a telephone usecase
* If we sent a notification event, we want the ui to show a ringing state
*/
waitForCallPickup?: boolean;
}
// How long we wait after a focus switch before showing the real participant
// list again
const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
// This is the number of participants that we think constitutes a "small" call
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
@@ -555,6 +572,17 @@ export class CallViewModel extends ViewModel {
)
.pipe(pauseWhen(this.pretendToBeDisconnected$));
private readonly memberships$ = this.scope.behavior(
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(
startWith(null),
pauseWhen(this.pretendToBeDisconnected$),
map(() => this.matrixRTCSession.memberships),
),
);
/**
* Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members
@@ -564,18 +592,17 @@ export class CallViewModel extends ViewModel {
// than on Chrome/Firefox). This means it is important that we multicast the result so that we
// don't do this work more times than we need to. This is achieved by converting to a behavior:
public readonly memberDisplaynames$ = this.scope.behavior(
merge(
// Handle call membership changes.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
),
// Handle room membership changes (and displayname updates)
fromEvent(this.matrixRoom, RoomStateEvent.Members),
).pipe(
startWith(null),
map(() => {
const memberships = this.matrixRTCSession.memberships;
// React to call memberships and also display name updates
// (calculateDisplayName implicitly depends on the room member data)
combineLatest(
[
this.memberships$,
fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe(
startWith(null),
pauseWhen(this.pretendToBeDisconnected$),
),
],
(memberships, _members) => {
const displaynameMap = new Map<string, string>();
const room = this.matrixRoom;
@@ -597,8 +624,7 @@ export class CallViewModel extends ViewModel {
);
}
return displaynameMap;
}),
pauseWhen(this.pretendToBeDisconnected$),
},
),
);
@@ -628,13 +654,7 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants$,
observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value$,
// Also react to changes in the MatrixRTC session list.
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)),
this.memberships$,
showNonMemberTiles.value$,
]).pipe(
scan(
@@ -644,7 +664,7 @@ export class CallViewModel extends ViewModel {
remoteParticipants,
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
memberships,
showNonMemberTiles,
],
) => {
@@ -652,7 +672,7 @@ export class CallViewModel extends ViewModel {
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
const room = this.matrixRoom;
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
for (const rtcMember of memberships) {
const { member, id: livekitParticipantId } =
getRoomMemberFromRtcMember(rtcMember, room);
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
@@ -818,79 +838,118 @@ export class CallViewModel extends ViewModel {
),
);
/**
* This observable tracks the currently connected participants.
*
* - Each participant has one livekit connection
* - Each participant has a corresponding MatrixRTC membership state event
* - There can be multiple participants for one matrix user.
*/
public readonly participantChanges$ = this.userMedia$.pipe(
map((mediaItems) => mediaItems.map((m) => m.id)),
scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
(prev, ids) => {
const left = prev.ids.filter((id) => !ids.includes(id));
const joined = ids.filter((id) => !prev.ids.includes(id));
return { ids, joined, left };
},
{ ids: [], joined: [], left: [] },
public readonly joinSoundEffect$ = this.userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
current.length > prev.length,
),
);
/**
* This observable tracks the matrix users that are currently in the call.
* There can be just one matrix user with multiple participants (see also participantChanges$)
*/
public readonly matrixUserChanges$ = this.userMedia$.pipe(
map(
(mediaItems) =>
new Set(
mediaItems
.map((m) => m.vm.member?.userId)
.filter((id) => id !== undefined),
),
),
scan<
Set<string>,
{
userIds: Set<string>;
joinedUserIds: Set<string>;
leftUserIds: Set<string>;
}
>(
(prevState, userIds) => {
const left = new Set(
[...prevState.userIds].filter((id) => !userIds.has(id)),
);
const joined = new Set(
[...userIds].filter((id) => !prevState.userIds.has(id)),
);
return { userIds: userIds, joinedUserIds: joined, leftUserIds: left };
},
{ userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() },
),
);
public readonly allOthersLeft$ = this.matrixUserChanges$.pipe(
map(({ userIds, leftUserIds }) => {
if (!this.userId) {
logger.warn("Could not access user ID to compute allOthersLeft");
return false;
}
return (
userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0
);
}),
startWith(false),
distinctUntilChanged(),
);
public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe(
distinctUntilChanged(),
filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
public readonly leaveSoundEffect$ = this.userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
current.length < prev.length,
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
/**
* The number of participants currently in the call.
*
* - Each participant has a corresponding MatrixRTC membership state event
* - There can be multiple participants for one Matrix user if they join from
* multiple devices.
*/
public readonly participantCount$ = this.scope.behavior(
this.memberships$.pipe(map((ms) => ms.length)),
);
private readonly allOthersLeft$ = this.memberships$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.every((m) => m.sender === this.userId) &&
prev.some((m) => m.sender !== this.userId),
),
map(() => {}),
take(1),
);
public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft
? this.allOthersLeft$
: NEVER;
/**
* Emits whenever the RTC session tells us that it intends to ring for a given
* duration.
*/
private readonly beginRingingForMs$ = (
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.DidSendCallNotification,
) as Observable<[IRTCNotificationContent]>
)
// event.lifetime is expected to be in ms
.pipe(map(([notificationEvent]) => notificationEvent?.lifetime ?? 0));
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
private readonly someoneElseJoined$ = this.memberships$.pipe(
map((ms) => ms.some((m) => m.sender !== this.userId)),
);
/**
* The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*/
public readonly callPickupState$ = this.options.waitForCallPickup
? this.scope.behavior<"unknown" | "ringing" | "timeout" | "success">(
concat(
concat(
// We don't know if the RTC session decides to send a notify event
// yet. It will only be known once we sent our own membership and
// know we were the first one to join.
of("unknown" as const),
// Once we get the signal to begin ringing:
this.beginRingingForMs$.pipe(
take(1),
switchMap((lifetime) =>
lifetime === 0
? // If no lifetime, skip the ring state
EMPTY
: // Ring until lifetime ms have passed
timer(lifetime).pipe(
ignoreElements(),
startWith("ringing" as const),
),
),
),
// The notification lifetime has timed out, meaning ringing has
// likely stopped on all receiving clients.
of("timeout" as const),
NEVER,
).pipe(
takeUntil(this.someoneElseJoined$.pipe(filter((joined) => joined))),
),
of("success" as const),
),
)
: constant(null);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/

4
yarn.lock
View File

@@ -10280,7 +10280,7 @@ __metadata:
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
version: 37.13.0
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=3a33c658bbcb8ce8791ec066db899f2571f5c52f"
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=c4c7f945141e142e6f846b243c33c4af97a9a44b"
dependencies:
"@babel/runtime": "npm:^7.12.5"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0"
@@ -10296,7 +10296,7 @@ __metadata:
sdp-transform: "npm:^2.14.1"
unhomoglyph: "npm:^1.0.6"
uuid: "npm:11"
checksum: 10c0/1db0d39cfbe4f1c69c8acda0ea7580a4819fc47a7d4bff057382e33e72d9a610f8c03043a6c00bc647dfdc2815aa643c69d25022fb759342a92b77e1841524f1
checksum: 10c0/caa4b8a6d924ac36a21773dc2c8be6cb6b658a9feaabccdb24426719c563ac2cfe4778abb86f0889854ae36fc7ba02a6ed39acdbc0b73fdc31ce9a9789e7f36a
languageName: node
linkType: hard