diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 30019d36..f2ee5d4d 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -216,6 +216,17 @@ export interface UrlConfiguration { * This is one part to make the call matrixRTC session behave like a telephone call. */ autoLeaveWhenOthersLeft: boolean; + + /** + * If the client should show behave like it is awaiting an answer if a notification was sent. + * This is a no-op if not combined with sendNotificationType. + * + * This entails: + * - show ui that it is awaiting an answer + * - play a sound that indicates that it is awaiting an answer + * - auto-dismiss the call widget once the notification lifetime expires on the receivers side. + */ + awaitingAnswer: boolean; } // If you need to add a new flag to this interface, prefer a name that describes @@ -442,6 +453,7 @@ export const getUrlParams = ( "ring", "notification", ]), + awaitingAnswer: parser.getFlag("showAwaitingAnswerFeedback"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), }; diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index ea57bd10..63fc942f 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -453,7 +453,6 @@ export const GroupCallView: FC = ({ matrixInfo={matrixInfo} rtcSession={rtcSession as MatrixRTCSession} matrixRoom={room} - participantCount={participantCount} onLeave={onLeave} header={header} muteStates={muteStates} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 4e3229a5..16e03987 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -216,7 +216,6 @@ export interface InCallViewProps { matrixRoom: MatrixRoom; livekitRoom: LivekitRoom; muteStates: MuteStates; - participantCount: number; /** Function to call when the user explicitly ends the call */ onLeave: () => void; header: HeaderStyle; @@ -233,7 +232,6 @@ export const InCallView: FC = ({ matrixRoom, livekitRoom, muteStates, - participantCount, onLeave, header: headerStyle, connState, @@ -312,6 +310,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); const layout = useBehavior(vm.layout$); diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 30fbad36..a4f5d2a3 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished, it } from "vitest"; +import { test, vi, onTestFinished, it, describe } from "vitest"; import EventEmitter from "events"; import { BehaviorSubject, @@ -32,6 +32,9 @@ import { Status, type CallMembership, type MatrixRTCSession, + type IRTCNotificationContent, + type ICallNotifyContent, + MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -1228,6 +1231,215 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option }); }); +describe("waitForNotificationAnswer$", () => { + test("unknown -> ringing -> timeout when notified and nobody joins", () => { + withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + // No one ever joins (only local user) + withCallViewModel( + { + remoteParticipants$: scope.behavior(hot("a", { a: [] }), []), + rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []), + connectionState$: of(ConnectionState.Connected), + speaking: new Map(), + mediaDevices: mockMediaDevices({}), + }, + (vm, rtcSession) => { + // Fire a call notification at 10ms with lifetime 30ms + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 30 } as unknown as IRTCNotificationContent, + {} as unknown as ICallNotifyContent, + ); + }, + }); + + expectObservable(vm.waitForNotificationAnswer$).toBe( + "a 9ms b 29ms c", + { a: "unknown", b: "ringing", c: "timeout" }, + ); + }, + { + waitForNotificationAnswer: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("ringing -> success if someone joins before timeout", () => { + withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const remote$ = scope.behavior( + hot("a--b", { a: [], b: [aliceParticipant] }), + [], + ); + const rtc$ = scope.behavior( + hot("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + [], + ); + + withCallViewModel( + { + remoteParticipants$: remote$, + rtcMembers$: rtc$, + connectionState$: of(ConnectionState.Connected), + speaking: new Map(), + mediaDevices: mockMediaDevices({}), + }, + (vm, rtcSession) => { + // Notify at 5ms so we enter ringing, then success at 20ms + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 100 } as unknown as IRTCNotificationContent, + {} as unknown as ICallNotifyContent, + ); + }, + }); + + expectObservable(vm.waitForNotificationAnswer$).toBe("a 2ms c", { + a: "unknown", + c: "success", + }); + }, + { + waitForNotificationAnswer: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("success when someone joins before we notify", () => { + withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + // Join at 10ms, notify later at 20ms (state should stay success) + const remote$ = scope.behavior( + hot("a-b", { a: [], b: [aliceParticipant] }), + [], + ); + const rtc$ = scope.behavior( + hot("a-b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + [], + ); + + withCallViewModel( + { + remoteParticipants$: remote$, + rtcMembers$: rtc$, + connectionState$: of(ConnectionState.Connected), + speaking: new Map(), + mediaDevices: mockMediaDevices({}), + }, + (vm, rtcSession) => { + schedule(" 20ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 50 } as unknown as IRTCNotificationContent, + {} as unknown as ICallNotifyContent, + ); + }, + }); + expectObservable(vm.waitForNotificationAnswer$).toBe("a 1ms b", { + a: "unknown", + b: "success", + }); + }, + { + waitForNotificationAnswer: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("notify without lifetime -> immediate timeout", () => { + withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + withCallViewModel( + { + remoteParticipants$: scope.behavior(hot("a", { a: [] }), []), + rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []), + connectionState$: of(ConnectionState.Connected), + speaking: new Map(), + mediaDevices: mockMediaDevices({}), + }, + (vm, rtcSession) => { + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 0 } as unknown as IRTCNotificationContent, // no lifetime + {} as unknown as ICallNotifyContent, + ); + }, + }); + expectObservable(vm.waitForNotificationAnswer$).toBe("a 9ms b", { + a: "unknown", + b: "timeout", + }); + }, + { + waitForNotificationAnswer: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("stays null when waitForNotificationAnswer=false", () => { + withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + const remote$ = scope.behavior( + hot("a--b", { a: [], b: [aliceParticipant] }), + [], + ); + const rtc$ = scope.behavior( + hot("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + [], + ); + + withCallViewModel( + { + remoteParticipants$: remote$, + rtcMembers$: rtc$, + connectionState$: of(ConnectionState.Connected), + speaking: new Map(), + mediaDevices: mockMediaDevices({}), + }, + (vm, rtcSession) => { + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 30 } as unknown as IRTCNotificationContent, + {} as unknown as ICallNotifyContent, + ); + }, + }); + expectObservable(vm.waitForNotificationAnswer$).toBe("(n)", { + n: null, + }); + }, + { + waitForNotificationAnswer: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); +}); + test("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { getUrlParams.mockReturnValue({ controlledAudioDevices: true }); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index d7bf1812..f22269c0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -53,6 +53,8 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + type ICallNotifyContent, + type IRTCNotificationContent, type MatrixRTCSession, MatrixRTCSessionEvent, MembershipManagerEvent, @@ -110,7 +112,13 @@ import { type Behavior } from "./Behavior"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForNotificationAnswer?: boolean; } + // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; @@ -427,7 +435,14 @@ export class CallViewModel extends ViewModel { MembershipManagerEvent.StatusChanged, ).pipe( startWith(null), - map(() => this.matrixRTCSession.membershipStatus === Status.Connected), + map( + () => + ( + this.matrixRTCSession as unknown as { + membershipStatus?: Status; + } + ).membershipStatus === Status.Connected, + ), ), // Also watch out for warnings that we've likely hit a timeout and our // delayed leave event is being sent (this condition is here because it @@ -438,7 +453,11 @@ export class CallViewModel extends ViewModel { MembershipManagerEvent.ProbablyLeft, ).pipe( startWith(null), - map(() => this.matrixRTCSession.probablyLeft !== true), + map( + () => + (this.matrixRTCSession as unknown as { probablyLeft?: boolean }) + .probablyLeft !== true, + ), ), ), ); @@ -825,49 +844,68 @@ export class CallViewModel extends ViewModel { * - Each participant has a corresponding MatrixRTC membership state event * - There can be multiple participants for one matrix user. */ - public readonly participantChanges$ = this.userMedia$.pipe( - map((mediaItems) => mediaItems.map((m) => m.id)), - scan( - (prev, ids) => { - const left = prev.ids.filter((id) => !ids.includes(id)); - const joined = ids.filter((id) => !prev.ids.includes(id)); - return { ids, joined, left }; - }, - { ids: [], joined: [], left: [] }, + public readonly participantChanges$ = this.scope.behavior( + this.userMedia$.pipe( + map((mediaItems) => mediaItems.map((m) => m.id)), + scan( + (prev, ids) => { + const left = prev.ids.filter((id) => !ids.includes(id)); + const joined = ids.filter((id) => !prev.ids.includes(id)); + return { ids, joined, left }; + }, + { ids: [], joined: [], left: [] }, + ), ), ); + /** + * The number of participants currently in the call. + * + * - Each participant has one livekit connection + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one matrix user. + */ + public readonly participantCount$ = this.scope.behavior( + this.participantChanges$.pipe(map(({ ids }) => ids.length)), + ); + /** * This observable tracks the matrix users that are currently in the call. * There can be just one matrix user with multiple participants (see also participantChanges$) */ - public readonly matrixUserChanges$ = this.userMedia$.pipe( - map( - (mediaItems) => - new Set( - mediaItems - .map((m) => m.vm.member?.userId) - .filter((id) => id !== undefined), - ), - ), - scan< - Set, - { - userIds: Set; - joinedUserIds: Set; - leftUserIds: Set; - } - >( - (prevState, userIds) => { - const left = new Set( - [...prevState.userIds].filter((id) => !userIds.has(id)), - ); - const joined = new Set( - [...userIds].filter((id) => !prevState.userIds.has(id)), - ); - return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; - }, - { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, + public readonly matrixUserChanges$ = this.scope.behavior( + this.userMedia$.pipe( + map( + (mediaItems) => + new Set( + mediaItems + .map((m) => m.vm.member?.userId) + .filter((id) => id !== undefined), + ), + ), + scan< + Set, + { + userIds: Set; + joinedUserIds: Set; + leftUserIds: Set; + } + >( + (prevState, userIds) => { + const left = new Set( + [...prevState.userIds].filter((id) => !userIds.has(id)), + ); + const joined = new Set( + [...userIds].filter((id) => !prevState.userIds.has(id)), + ); + return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; + }, + { + userIds: new Set(), + joinedUserIds: new Set(), + leftUserIds: new Set(), + }, + ), ), ); @@ -891,6 +929,84 @@ export class CallViewModel extends ViewModel { map(() => {}), ); + /** + * "unknown": We don't know if the RTC session decides to send a notify event yet. + * It will only be known once we sent our own membership and know we were the first one to join. + * "ringing": The notification event was sent. + * "ringEnded": The notification events lifetime has timed out -> ringing stopped on all receiving clients. + */ + private readonly notificationEventIsRingingOthers$: Observable< + "unknown" | "ringing" | "ringEnded" | null + > = fromEvent<[IRTCNotificationContent, ICallNotifyContent]>( + this.matrixRTCSession, + MatrixRTCSessionEvent.DidSendCallNotification, + ).pipe( + switchMap(([notificationEvent]) => { + // event.lifetime is expected to be in ms + const lifetime = notificationEvent?.lifetime ?? 0; + if (lifetime > 0) { + // Emit true immediately, then false after lifetime ms + return concat( + of<"ringing" | null>("ringing"), + timer(lifetime).pipe(map((): "ringEnded" | null => "ringEnded")), + ); + } + // If no lifetime, just emit true once + return of(null); + }), + startWith("unknown" as "unknown" | null), + ); + + /** + * If some other matrix user has joined the call. It can start with true if there are already multiple matrix users. + */ + private readonly someoneElseJoined$ = this.matrixUserChanges$.pipe( + scan( + (someoneJoined, { joinedUserIds }) => + someoneJoined || [...joinedUserIds].some((id) => id !== this.userId), + false, + ), + startWith(this.matrixUserChanges$.value.userIds.size > 1), + ); + + /** + * The current waiting for answer state of the call. + * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). + * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. + * Then we can conclude if we were the first one to join or not. + * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. + * The call failed. If desired this can be used as a trigger to exit the call. + * - "success": Someone else joined. The call is in a normal state. Stop audiovisual feedback. + * - null: EC is configured to never show any waiting for answer state. + */ + public readonly waitForNotificationAnswer$: Behavior< + "unknown" | "ringing" | "timeout" | "success" | null + > = this.scope.behavior( + combineLatest([ + this.notificationEventIsRingingOthers$, + this.someoneElseJoined$, + ]).pipe( + map(([isRingingOthers, someoneJoined]) => { + // Never enter waiting for answer state if the app is not configured with waitingForAnswer. + if (!this.options.waitForNotificationAnswer) return null; + // As soon as someone joins, we can consider the call "wait for answer" successful + if (someoneJoined) return "success"; + + switch (isRingingOthers) { + case "unknown": + return "unknown"; + case "ringing": + return "ringing"; + case "ringEnded": + return "timeout"; + default: + return "timeout"; + } + }), + distinctUntilChanged(), + ), + ); + /** * List of MediaItems that we want to display, that are of type ScreenShare */ diff --git a/yarn.lock b/yarn.lock index 0d2cf54a..15ed17d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10278,9 +10278,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": - version: 37.13.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=3a33c658bbcb8ce8791ec066db899f2571f5c52f" +"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0" @@ -10296,9 +10296,8 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/1db0d39cfbe4f1c69c8acda0ea7580a4819fc47a7d4bff057382e33e72d9a610f8c03043a6c00bc647dfdc2815aa643c69d25022fb759342a92b77e1841524f1 languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": version: 1.13.1