Merge pull request #3466 from element-hq/toger5/waitForNotificationAnswer

Add dialing/ringing state to CallViewModel (`callPickupState$`)

docs/url-params.md

@@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
 | `theme`                                                 | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast`                                 | No, defaults to `dark`                                                                            | No, defaults to `dark`       | UI theme to use.                                                                                                                                                                                                                                                                                                                                                                                    |
 | `viaServers`                                            | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable                                                                                    | No                           | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver.                                                                                                                                                                                                                                                                                             |
 | `sendNotificationType`                                  | `ring` or `notification`                                                                             | No                                                                                                | No                           | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call.                                                                                                                                                                                                                                                                                           |
+| `autoLeaveWhenOthersLeft`                               | `true` or `false`                                                                                    | No, defaults to `false`                                                                           | No, defaults to `false`      | Whether the app should automatically leave the call when there is no one left in the call.                                                                                                                                                                                                                                                                                                          |
+| `waitForCallPickup`                                     | `true` or `false`                                                                                    | No, defaults to `false`                                                                           | No, defaults to `false`      | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires.                                                                                                                                                                                                                                |
 
 ### Widget-only parameters
 

src/UrlParams.ts

@@ -216,6 +216,17 @@ export interface UrlConfiguration {
    * This is one part to make the call matrixRTC session behave like a telephone call.
    */
   autoLeaveWhenOthersLeft: boolean;
+
+  /**
+   * If the client should behave like it is awaiting an answer if a notification was sent (wait for call pick up).
+   * This is a no-op if not combined with sendNotificationType.
+   *
+   * This entails:
+   *  - show ui that it is awaiting an answer
+   *  - play a sound that indicates that it is awaiting an answer
+   *  - auto-dismiss the call widget once the notification lifetime expires on the receivers side.
+   */
+  waitForCallPickup: boolean;
 }
 
 // If you need to add a new flag to this interface, prefer a name that describes
@@ -347,6 +358,7 @@ export const getUrlParams = (
     returnToLobby: false,
     sendNotificationType: "notification" as RTCNotificationType,
     autoLeaveWhenOthersLeft: false,
+    waitForCallPickup: false,
   };
   switch (intent) {
     case UserIntent.StartNewCall:
@@ -366,6 +378,7 @@ export const getUrlParams = (
         ...inAppDefault,
         skipLobby: true,
         autoLeaveWhenOthersLeft: true,
+        waitForCallPickup: true,
       };
       break;
     case UserIntent.JoinExistingCallDM:
@@ -391,6 +404,7 @@ export const getUrlParams = (
         returnToLobby: false,
         sendNotificationType: undefined,
         autoLeaveWhenOthersLeft: false,
+        waitForCallPickup: false,
       };
   }
 
@@ -442,6 +456,7 @@ export const getUrlParams = (
       "ring",
       "notification",
     ]),
+    waitForCallPickup: parser.getFlag("waitForCallPickup"),
     autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
   };
 

src/livekit/useECConnectionState.test.tsx

@@ -94,7 +94,7 @@ describe("Leaking connection prevention", () => {
   test("Should cancel pending connections when the component is unmounted", async () => {
     const connectCall = vi.fn();
     const pendingConnection = Promise.withResolvers<void>();
-    // let pendingDisconnection = defer<void>()
+    // let pendingDisconnection = Promise.withResolvers<void>()
     const disconnectMock = vi.fn();
 
     const mockRoom = {
@@ -142,7 +142,7 @@ describe("Leaking connection prevention", () => {
   test("Should cancel about to open but not yet opened connection", async () => {
     const createTracksCall = vi.fn();
     const pendingCreateTrack = Promise.withResolvers<void>();
-    // let pendingDisconnection = defer<void>()
+    // let pendingDisconnection = Promise.withResolvers<void>()
     const disconnectMock = vi.fn();
     const connectMock = vi.fn();
 

src/room/CallEventAudioRenderer.test.tsx

@@ -19,10 +19,7 @@ import { act } from "react";
 import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
 
 import { mockRtcMembership } from "../utils/test";
-import {
+import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
-  CallEventAudioRenderer,
-  MAX_PARTICIPANT_COUNT_FOR_SOUND,
-} from "./CallEventAudioRenderer";
 import { useAudioContext } from "../useAudioContext";
 import { prefetchSounds } from "../soundUtils";
 import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
@@ -33,6 +30,7 @@ import {
   local,
   localRtcMember,
 } from "../utils/test-fixtures";
+import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel";
 
 vitest.mock("../useAudioContext");
 vitest.mock("../soundUtils");
@@ -158,6 +156,7 @@ test("should not play a sound when a hand raise is retracted", () => {
   ]);
   render(<CallEventAudioRenderer vm={vm} />);
 
+  playSound.mockClear();
   act(() => {
     handRaisedSubject$.next({
       ["foo"]: {
@@ -172,7 +171,7 @@ test("should not play a sound when a hand raise is retracted", () => {
       },
     });
   });
-  expect(playSound).toHaveBeenCalledTimes(2);
+  expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand");
   act(() => {
     handRaisedSubject$.next({
       ["foo"]: {
@@ -182,5 +181,5 @@ test("should not play a sound when a hand raise is retracted", () => {
       },
     });
   });
-  expect(playSound).toHaveBeenCalledTimes(2);
+  expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand");
 });

src/room/CallEventAudioRenderer.tsx

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { type ReactNode, useEffect } from "react";
-import { filter, interval, throttle } from "rxjs";
 
 import { type CallViewModel } from "../state/CallViewModel";
 import joinCallSoundMp3 from "../sound/join_call.mp3";
@@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext";
 import { prefetchSounds } from "../soundUtils";
 import { useLatest } from "../useLatest";
 
-// Do not play any sounds if the participant count has exceeded this
-// number.
-export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
-export const THROTTLE_SOUND_EFFECT_MS = 500;
-
 export const callEventAudioSounds = prefetchSounds({
   join: {
     mp3: joinCallSoundMp3,
@@ -60,37 +54,18 @@ export function CallEventAudioRenderer({
   const audioEngineRef = useLatest(audioEngineCtx);
 
   useEffect(() => {
-    const joinSub = vm.participantChanges$
+    const joinSub = vm.joinSoundEffect$.subscribe(
-      .pipe(
+      () => void audioEngineRef.current?.playSound("join"),
-        filter(
+    );
-          ({ joined, ids }) =>
+    const leftSub = vm.leaveSoundEffect$.subscribe(
-            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
+      () => void audioEngineRef.current?.playSound("left"),
-        ),
+    );
-        throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
+    const handRaisedSub = vm.newHandRaised$.subscribe(
-      )
+      () => void audioEngineRef.current?.playSound("raiseHand"),
-      .subscribe(() => {
+    );
-        void audioEngineRef.current?.playSound("join");
+    const screenshareSub = vm.newScreenShare$.subscribe(
-      });
+      () => void audioEngineRef.current?.playSound("screenshareStarted"),
-
+    );
-    const leftSub = vm.participantChanges$
-      .pipe(
-        filter(
-          ({ ids, left }) =>
-            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
-        ),
-        throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
-      )
-      .subscribe(() => {
-        void audioEngineRef.current?.playSound("left");
-      });
-
-    const handRaisedSub = vm.newHandRaised$.subscribe(() => {
-      void audioEngineRef.current?.playSound("raiseHand");
-    });
-
-    const screenshareSub = vm.newScreenShare$.subscribe(() => {
-      void audioEngineRef.current?.playSound("screenshareStarted");
-    });
 
     return (): void => {
       joinSub.unsubscribe();

src/room/GroupCallView.tsx

@@ -453,7 +453,6 @@ export const GroupCallView: FC<Props> = ({
           matrixInfo={matrixInfo}
           rtcSession={rtcSession as MatrixRTCSession}
           matrixRoom={room}
-          participantCount={participantCount}
           onLeave={onLeave}
           header={header}
           muteStates={muteStates}

src/room/InCallView.test.tsx

@@ -177,7 +177,6 @@ function createInCallView(): RenderResult & {
                 }}
                 matrixRoom={room}
                 livekitRoom={livekitRoom}
-                participantCount={0}
                 onLeave={function (): void {
                   throw new Error("Function not implemented.");
                 }}

src/room/InCallView.tsx

@@ -216,7 +216,6 @@ export interface InCallViewProps {
   matrixRoom: MatrixRoom;
   livekitRoom: LivekitRoom;
   muteStates: MuteStates;
-  participantCount: number;
   /** Function to call when the user explicitly ends the call */
   onLeave: () => void;
   header: HeaderStyle;
@@ -233,7 +232,6 @@ export const InCallView: FC<InCallViewProps> = ({
   matrixRoom,
   livekitRoom,
   muteStates,
-  participantCount,
   onLeave,
   header: headerStyle,
   connState,
@@ -312,6 +310,7 @@ export const InCallView: FC<InCallViewProps> = ({
     () => void toggleRaisedHand(),
   );
 
+  const participantCount = useBehavior(vm.participantCount$);
   const reconnecting = useBehavior(vm.reconnecting$);
   const windowMode = useBehavior(vm.windowMode$);
   const layout = useBehavior(vm.layout$);
@@ -322,7 +321,7 @@ export const InCallView: FC<InCallViewProps> = ({
   const showFooter = useBehavior(vm.showFooter$);
   const earpieceMode = useBehavior(vm.earpieceMode$);
   const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
-  useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave);
+  useSubscription(vm.autoLeave$, onLeave);
 
   // Ideally we could detect taps by listening for click events and checking
   // that the pointerType of the event is "touch", but this isn't yet supported

src/room/__snapshots__/InCallView.test.tsx.snap

@@ -49,6 +49,34 @@ exports[`InCallView > rendering > renders 1`] = `
               </svg>
             </span>
           </div>
+          <div
+            class="participantsLine"
+          >
+            <svg
+              aria-label="Participants"
+              fill="currentColor"
+              height="20"
+              viewBox="0 0 24 24"
+              width="20"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                d="M9.175 13.825Q10.35 15 12 15t2.825-1.175T16 11t-1.175-2.825T12 7 9.175 8.175 8 11t1.175 2.825m4.237-1.412A1.93 1.93 0 0 1 12 13q-.825 0-1.412-.588A1.93 1.93 0 0 1 10 11q0-.825.588-1.412A1.93 1.93 0 0 1 12 9q.825 0 1.412.588Q14 10.175 14 11t-.588 1.412"
+              />
+              <path
+                d="M22 12c0 5.523-4.477 10-10 10S2 17.523 2 12 6.477 2 12 2s10 4.477 10 10m-2 0a8 8 0 1 0-16 0 8 8 0 0 0 16 0"
+              />
+              <path
+                d="M16.23 18.792a13 13 0 0 0-1.455-.455 11.6 11.6 0 0 0-5.55 0q-.73.18-1.455.455a8 8 0 0 1-1.729-1.454q1.336-.618 2.709-.95A13.8 13.8 0 0 1 12 16q1.65 0 3.25.387 1.373.333 2.709.95a8 8 0 0 1-1.73 1.455"
+              />
+            </svg>
+            <span
+              class="_typography_6v6n8_153 _font-body-sm-medium_6v6n8_41"
+              data-testid="roomHeader_participants_count"
+            >
+              2
+            </span>
+          </div>
         </div>
       </div>
       <div

src/state/CallViewModel.test.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { test, vi, onTestFinished, it } from "vitest";
+import { test, vi, onTestFinished, it, describe } from "vitest";
 import EventEmitter from "events";
 import {
   BehaviorSubject,
@@ -32,6 +32,9 @@ import {
   Status,
   type CallMembership,
   type MatrixRTCSession,
+  type IRTCNotificationContent,
+  type ICallNotifyContent,
+  MatrixRTCSessionEvent,
 } from "matrix-js-sdk/lib/matrixrtc";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
 
@@ -308,7 +311,7 @@ function withCallViewModel(
 
   const roomEventSelectorSpy = vi
     .spyOn(ComponentsCore, "roomEventSelector")
-    .mockImplementation((room, eventType) => of());
+    .mockImplementation((_room, _eventType) => of());
 
   const livekitRoom = mockLivekitRoom(
     { localParticipant },
@@ -1068,9 +1071,9 @@ it("should rank raised hands above video feeds and below speakers and presenters
 });
 
 function nooneEverThere$<T>(
-  hot: (marbles: string, values: Record<string, T[]>) => Observable<T[]>,
+  behavior: (marbles: string, values: Record<string, T[]>) => Behavior<T[]>,
-): Observable<T[]> {
+): Behavior<T[]> {
-  return hot("a-b-c-d", {
+  return behavior("a-b-c-d", {
     a: [], // Start empty
     b: [], // Alice joins
     c: [], // Alice still there
@@ -1079,12 +1082,12 @@ function nooneEverThere$<T>(
 }
 
 function participantJoinLeave$(
-  hot: (
+  behavior: (
     marbles: string,
     values: Record<string, RemoteParticipant[]>,
-  ) => Observable<RemoteParticipant[]>,
+  ) => Behavior<RemoteParticipant[]>,
-): Observable<RemoteParticipant[]> {
+): Behavior<RemoteParticipant[]> {
-  return hot("a-b-c-d", {
+  return behavior("a-b-c-d", {
     a: [], // Start empty
     b: [aliceParticipant], // Alice joins
     c: [aliceParticipant], // Alice still there
@@ -1093,12 +1096,12 @@ function participantJoinLeave$(
 }
 
 function rtcMemberJoinLeave$(
-  hot: (
+  behavior: (
     marbles: string,
     values: Record<string, CallMembership[]>,
-  ) => Observable<CallMembership[]>,
+  ) => Behavior<CallMembership[]>,
-): Observable<CallMembership[]> {
+): Behavior<CallMembership[]> {
-  return hot("a-b-c-d", {
+  return behavior("a-b-c-d", {
     a: [localRtcMember], // Start empty
     b: [localRtcMember, aliceRtcMember], // Alice joins
     c: [localRtcMember, aliceRtcMember], // Alice still there
@@ -1106,47 +1109,15 @@ function rtcMemberJoinLeave$(
   });
 }
 
-test("allOthersLeft$ emits only when someone joined and then all others left", () => {
+test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
+  withTestScheduler(({ behavior, expectObservable }) => {
-    // Test scenario 1: No one ever joins - should only emit initial false and never emit again
-    withCallViewModel(
-      { remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) },
-      (vm) => {
-        expectObservable(vm.allOthersLeft$).toBe("n------", { n: false });
-      },
-    );
-  });
-});
-
-test("allOthersLeft$ emits true when someone joined and then all others left", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
     withCallViewModel(
       {
-        remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
+        remoteParticipants$: participantJoinLeave$(behavior),
-        rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
+        rtcMembers$: rtcMemberJoinLeave$(behavior),
       },
       (vm) => {
-        expectObservable(vm.allOthersLeft$).toBe(
+        expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined });
-          "n-----u", // false initially, then at frame 6: true then false emissions in same frame
-          { n: false, u: true }, // map(() => {})
-        );
-      },
-    );
-  });
-});
-
-test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
-    withCallViewModel(
-      {
-        remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
-        rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
-      },
-      (vm) => {
-        expectObservable(vm.autoLeaveWhenOthersLeft$).toBe(
-          "------e", // false initially, then at frame 6: true then false emissions in same frame
-          { e: undefined },
-        );
       },
       {
         autoLeaveWhenOthersLeft: true,
@@ -1156,15 +1127,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is
   });
 });
 
-test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
+test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
+  withTestScheduler(({ behavior, expectObservable }) => {
     withCallViewModel(
       {
-        remoteParticipants$: scope.behavior(nooneEverThere$(hot), []),
+        remoteParticipants$: nooneEverThere$(behavior),
-        rtcMembers$: scope.behavior(nooneEverThere$(hot), []),
+        rtcMembers$: nooneEverThere$(behavior),
       },
       (vm) => {
-        expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
+        expectObservable(vm.autoLeave$).toBe("-");
       },
       {
         autoLeaveWhenOthersLeft: true,
@@ -1174,15 +1145,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena
   });
 });
 
-test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
+test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
+  withTestScheduler(({ behavior, expectObservable }) => {
     withCallViewModel(
       {
-        remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
+        remoteParticipants$: participantJoinLeave$(behavior),
-        rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
+        rtcMembers$: rtcMemberJoinLeave$(behavior),
       },
       (vm) => {
-        expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
+        expectObservable(vm.autoLeave$).toBe("-");
       },
       {
         autoLeaveWhenOthersLeft: false,
@@ -1192,31 +1163,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option
   });
 });
 
-test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
+test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
-  withTestScheduler(({ hot, expectObservable, scope }) => {
+  withTestScheduler(({ behavior, expectObservable }) => {
     withCallViewModel(
       {
-        remoteParticipants$: scope.behavior(
+        remoteParticipants$: behavior("a-b-c-d", {
-          hot("a-b-c-d", {
+          a: [], // Alone
-            a: [], // Alone
+          b: [aliceParticipant], // Alice joins
-            b: [aliceParticipant], // Alice joins
+          c: [aliceParticipant],
-            c: [aliceParticipant],
+          d: [], // Local joins with a second device
-            d: [], // Local joins with a second device
+        }),
-          }),
+        rtcMembers$: behavior("a-b-c-d", {
-          [], //Alice leaves
+          a: [localRtcMember], // Start empty
-        ),
+          b: [localRtcMember, aliceRtcMember], // Alice joins
-        rtcMembers$: scope.behavior(
+          c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
-          hot("a-b-c-d", {
+          d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
-            a: [localRtcMember], // Start empty
+        }),
-            b: [localRtcMember, aliceRtcMember], // Alice joins
-            c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
-            d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
-          }),
-          [],
-        ),
       },
       (vm) => {
-        expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", {
+        expectObservable(vm.autoLeave$).toBe("------(e|)", {
           e: undefined,
         });
       },
@@ -1228,6 +1193,196 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option
   });
 });
 
+describe("waitForCallPickup$", () => {
+  test("unknown -> ringing -> timeout when notified and nobody joins", () => {
+    withTestScheduler(({ schedule, expectObservable }) => {
+      // No one ever joins (only local user)
+      withCallViewModel(
+        { remoteParticipants$: constant([]) },
+        (vm, rtcSession) => {
+          // Fire a call notification at 10ms with lifetime 30ms
+          schedule("          10ms r", {
+            r: () => {
+              rtcSession.emit(
+                MatrixRTCSessionEvent.DidSendCallNotification,
+                { lifetime: 30 } as unknown as {
+                  event_id: string;
+                } & IRTCNotificationContent,
+                {} as unknown as { event_id: string } & ICallNotifyContent,
+              );
+            },
+          });
+
+          expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", {
+            a: "unknown",
+            b: "ringing",
+            c: "timeout",
+          });
+        },
+        {
+          waitForCallPickup: true,
+          encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        },
+      );
+    });
+  });
+
+  test("ringing -> success if someone joins before timeout", () => {
+    withTestScheduler(({ behavior, schedule, expectObservable }) => {
+      // Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
+      withCallViewModel(
+        {
+          remoteParticipants$: behavior("a 19ms b", {
+            a: [],
+            b: [aliceParticipant],
+          }),
+          rtcMembers$: behavior("a 19ms b", {
+            a: [localRtcMember],
+            b: [localRtcMember, aliceRtcMember],
+          }),
+        },
+        (vm, rtcSession) => {
+          // Notify at 5ms so we enter ringing, then success at 20ms
+          schedule("          5ms r", {
+            r: () => {
+              rtcSession.emit(
+                MatrixRTCSessionEvent.DidSendCallNotification,
+                { lifetime: 100 } as unknown as {
+                  event_id: string;
+                } & IRTCNotificationContent,
+                {} as unknown as {
+                  event_id: string;
+                } & ICallNotifyContent,
+              );
+            },
+          });
+
+          expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", {
+            a: "unknown",
+            b: "ringing",
+            c: "success",
+          });
+        },
+        {
+          waitForCallPickup: true,
+          encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        },
+      );
+    });
+  });
+
+  test("success when someone joins before we notify", () => {
+    withTestScheduler(({ behavior, schedule, expectObservable }) => {
+      // Join at 10ms, notify later at 20ms (state should stay success)
+      withCallViewModel(
+        {
+          remoteParticipants$: behavior("a 9ms b", {
+            a: [],
+            b: [aliceParticipant],
+          }),
+          rtcMembers$: behavior("a 9ms b", {
+            a: [localRtcMember],
+            b: [localRtcMember, aliceRtcMember],
+          }),
+        },
+        (vm, rtcSession) => {
+          schedule("          20ms r", {
+            r: () => {
+              rtcSession.emit(
+                MatrixRTCSessionEvent.DidSendCallNotification,
+                { lifetime: 50 } as unknown as {
+                  event_id: string;
+                } & IRTCNotificationContent,
+                {} as unknown as {
+                  event_id: string;
+                } & ICallNotifyContent,
+              );
+            },
+          });
+          expectObservable(vm.callPickupState$).toBe("a 9ms b", {
+            a: "unknown",
+            b: "success",
+          });
+        },
+        {
+          waitForCallPickup: true,
+          encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        },
+      );
+    });
+  });
+
+  test("notify without lifetime -> immediate timeout", () => {
+    withTestScheduler(({ schedule, expectObservable }) => {
+      withCallViewModel(
+        {},
+        (vm, rtcSession) => {
+          schedule("          10ms r", {
+            r: () => {
+              rtcSession.emit(
+                MatrixRTCSessionEvent.DidSendCallNotification,
+                {} as unknown as {
+                  event_id: string;
+                } & IRTCNotificationContent, // no lifetime
+                {} as unknown as {
+                  event_id: string;
+                } & ICallNotifyContent,
+              );
+            },
+          });
+          expectObservable(vm.callPickupState$).toBe("a 9ms b", {
+            a: "unknown",
+            b: "timeout",
+          });
+        },
+        {
+          waitForCallPickup: true,
+          encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        },
+      );
+    });
+  });
+
+  test("stays null when waitForCallPickup=false", () => {
+    withTestScheduler(({ behavior, schedule, expectObservable }) => {
+      withCallViewModel(
+        {
+          remoteParticipants$: behavior("a--b", {
+            a: [],
+            b: [aliceParticipant],
+          }),
+          rtcMembers$: behavior("a--b", {
+            a: [localRtcMember],
+            b: [localRtcMember, aliceRtcMember],
+          }),
+        },
+        (vm, rtcSession) => {
+          schedule("          5ms r", {
+            r: () => {
+              rtcSession.emit(
+                MatrixRTCSessionEvent.DidSendCallNotification,
+                { lifetime: 30 } as unknown as {
+                  event_id: string;
+                } & IRTCNotificationContent,
+                {} as unknown as {
+                  event_id: string;
+                } & ICallNotifyContent,
+              );
+            },
+          });
+          expectObservable(vm.callPickupState$).toBe("(n)", {
+            n: null,
+          });
+        },
+        {
+          waitForCallPickup: false,
+          encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        },
+      );
+    });
+  });
+});
+
 test("audio output changes when toggling earpiece mode", () => {
   withTestScheduler(({ schedule, expectObservable }) => {
     getUrlParams.mockReturnValue({ controlledAudioDevices: true });

src/state/CallViewModel.ts

@@ -27,6 +27,7 @@ import {
 import {
   BehaviorSubject,
   EMPTY,
+  NEVER,
   type Observable,
   Subject,
   combineLatest,
@@ -35,10 +36,12 @@ import {
   filter,
   forkJoin,
   fromEvent,
+  ignoreElements,
   map,
   merge,
   mergeMap,
   of,
+  pairwise,
   race,
   scan,
   skip,
@@ -47,12 +50,15 @@ import {
   switchMap,
   switchScan,
   take,
+  takeUntil,
+  throttleTime,
   timer,
   withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
 import {
   type CallMembership,
+  type IRTCNotificationContent,
   type MatrixRTCSession,
   MatrixRTCSessionEvent,
   MembershipManagerEvent,
@@ -105,16 +111,27 @@ import { observeSpeaker$ } from "./observeSpeaker";
 import { shallowEquals } from "../utils/array";
 import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
 import { type MediaDevices } from "./MediaDevices";
-import { type Behavior } from "./Behavior";
+import { constant, type Behavior } from "./Behavior";
 
 export interface CallViewModelOptions {
   encryptionSystem: EncryptionSystem;
   autoLeaveWhenOthersLeft?: boolean;
+  /**
+   * If the call is started in a way where we want it to behave like a telephone usecase
+   * If we sent a notification event, we want the ui to show a ringing state
+   */
+  waitForCallPickup?: boolean;
 }
+
 // How long we wait after a focus switch before showing the real participant
 // list again
 const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
 
+// Do not play any sounds if the participant count has exceeded this
+// number.
+export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
+export const THROTTLE_SOUND_EFFECT_MS = 500;
+
 // This is the number of participants that we think constitutes a "small" call
 // on mobile. No spotlight tile should be shown below this threshold.
 const smallMobileCallThreshold = 3;
@@ -555,6 +572,17 @@ export class CallViewModel extends ViewModel {
     )
     .pipe(pauseWhen(this.pretendToBeDisconnected$));
 
+  private readonly memberships$ = this.scope.behavior(
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(
+      startWith(null),
+      pauseWhen(this.pretendToBeDisconnected$),
+      map(() => this.matrixRTCSession.memberships),
+    ),
+  );
+
   /**
    * Displaynames for each member of the call. This will disambiguate
    * any displaynames that clashes with another member. Only members
@@ -564,18 +592,17 @@ export class CallViewModel extends ViewModel {
   // than on Chrome/Firefox). This means it is important that we multicast the result so that we
   // don't do this work more times than we need to. This is achieved by converting to a behavior:
   public readonly memberDisplaynames$ = this.scope.behavior(
-    merge(
+    // React to call memberships and also display name updates
-      // Handle call membership changes.
+    // (calculateDisplayName implicitly depends on the room member data)
-      fromEvent(
+    combineLatest(
-        this.matrixRTCSession,
+      [
-        MatrixRTCSessionEvent.MembershipsChanged,
+        this.memberships$,
-      ),
+        fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe(
-      // Handle room membership changes (and displayname updates)
+          startWith(null),
-      fromEvent(this.matrixRoom, RoomStateEvent.Members),
+          pauseWhen(this.pretendToBeDisconnected$),
-    ).pipe(
+        ),
-      startWith(null),
+      ],
-      map(() => {
+      (memberships, _members) => {
-        const memberships = this.matrixRTCSession.memberships;
         const displaynameMap = new Map<string, string>();
         const room = this.matrixRoom;
 
@@ -597,8 +624,7 @@ export class CallViewModel extends ViewModel {
           );
         }
         return displaynameMap;
-      }),
+      },
-      pauseWhen(this.pretendToBeDisconnected$),
     ),
   );
 
@@ -628,13 +654,7 @@ export class CallViewModel extends ViewModel {
       this.remoteParticipants$,
       observeParticipantMedia(this.livekitRoom.localParticipant),
       duplicateTiles.value$,
-      // Also react to changes in the MatrixRTC session list.
+      this.memberships$,
-      // The session list will also be update if a room membership changes.
-      // No additional RoomState event listener needs to be set up.
-      fromEvent(
-        this.matrixRTCSession,
-        MatrixRTCSessionEvent.MembershipsChanged,
-      ).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)),
       showNonMemberTiles.value$,
     ]).pipe(
       scan(
@@ -644,7 +664,7 @@ export class CallViewModel extends ViewModel {
             remoteParticipants,
             { participant: localParticipant },
             duplicateTiles,
-            _membershipsChanged,
+            memberships,
             showNonMemberTiles,
           ],
         ) => {
@@ -652,7 +672,7 @@ export class CallViewModel extends ViewModel {
             function* (this: CallViewModel): Iterable<[string, MediaItem]> {
               const room = this.matrixRoom;
               // m.rtc.members are the basis for calculating what is visible in the call
-              for (const rtcMember of this.matrixRTCSession.memberships) {
+              for (const rtcMember of memberships) {
                 const { member, id: livekitParticipantId } =
                   getRoomMemberFromRtcMember(rtcMember, room);
                 const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
@@ -818,79 +838,118 @@ export class CallViewModel extends ViewModel {
     ),
   );
 
-  /**
+  public readonly joinSoundEffect$ = this.userMedia$.pipe(
-   * This observable tracks the currently connected participants.
+    pairwise(),
-   *
+    filter(
-   *  - Each participant has one livekit connection
+      ([prev, current]) =>
-   *  - Each participant has a corresponding MatrixRTC membership state event
+        current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
-   *  - There can be multiple participants for one matrix user.
+        current.length > prev.length,
-   */
-  public readonly participantChanges$ = this.userMedia$.pipe(
-    map((mediaItems) => mediaItems.map((m) => m.id)),
-    scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
-      (prev, ids) => {
-        const left = prev.ids.filter((id) => !ids.includes(id));
-        const joined = ids.filter((id) => !prev.ids.includes(id));
-        return { ids, joined, left };
-      },
-      { ids: [], joined: [], left: [] },
     ),
-  );
-
-  /**
-   * This observable tracks the matrix users that are currently in the call.
-   * There can be just one matrix user with multiple participants (see also participantChanges$)
-   */
-  public readonly matrixUserChanges$ = this.userMedia$.pipe(
-    map(
-      (mediaItems) =>
-        new Set(
-          mediaItems
-            .map((m) => m.vm.member?.userId)
-            .filter((id) => id !== undefined),
-        ),
-    ),
-    scan<
-      Set<string>,
-      {
-        userIds: Set<string>;
-        joinedUserIds: Set<string>;
-        leftUserIds: Set<string>;
-      }
-    >(
-      (prevState, userIds) => {
-        const left = new Set(
-          [...prevState.userIds].filter((id) => !userIds.has(id)),
-        );
-        const joined = new Set(
-          [...userIds].filter((id) => !prevState.userIds.has(id)),
-        );
-        return { userIds: userIds, joinedUserIds: joined, leftUserIds: left };
-      },
-      { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() },
-    ),
-  );
-
-  public readonly allOthersLeft$ = this.matrixUserChanges$.pipe(
-    map(({ userIds, leftUserIds }) => {
-      if (!this.userId) {
-        logger.warn("Could not access user ID to compute allOthersLeft");
-        return false;
-      }
-      return (
-        userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0
-      );
-    }),
-    startWith(false),
-    distinctUntilChanged(),
-  );
-
-  public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe(
-    distinctUntilChanged(),
-    filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false),
     map(() => {}),
+    throttleTime(THROTTLE_SOUND_EFFECT_MS),
   );
 
+  public readonly leaveSoundEffect$ = this.userMedia$.pipe(
+    pairwise(),
+    filter(
+      ([prev, current]) =>
+        current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
+        current.length < prev.length,
+    ),
+    map(() => {}),
+    throttleTime(THROTTLE_SOUND_EFFECT_MS),
+  );
+
+  /**
+   * The number of participants currently in the call.
+   *
+   *  - Each participant has a corresponding MatrixRTC membership state event
+   *  - There can be multiple participants for one Matrix user if they join from
+   *    multiple devices.
+   */
+  public readonly participantCount$ = this.scope.behavior(
+    this.memberships$.pipe(map((ms) => ms.length)),
+  );
+
+  private readonly allOthersLeft$ = this.memberships$.pipe(
+    pairwise(),
+    filter(
+      ([prev, current]) =>
+        current.every((m) => m.sender === this.userId) &&
+        prev.some((m) => m.sender !== this.userId),
+    ),
+    map(() => {}),
+    take(1),
+  );
+
+  public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft
+    ? this.allOthersLeft$
+    : NEVER;
+
+  /**
+   * Emits whenever the RTC session tells us that it intends to ring for a given
+   * duration.
+   */
+  private readonly beginRingingForMs$ = (
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.DidSendCallNotification,
+    ) as Observable<[IRTCNotificationContent]>
+  )
+    // event.lifetime is expected to be in ms
+    .pipe(map(([notificationEvent]) => notificationEvent?.lifetime ?? 0));
+
+  /**
+   * Whether some Matrix user other than ourself is joined to the call.
+   */
+  private readonly someoneElseJoined$ = this.memberships$.pipe(
+    map((ms) => ms.some((m) => m.sender !== this.userId)),
+  );
+
+  /**
+   * The current call pickup state of the call.
+   *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
+   *     Then we can conclude if we were the first one to join or not.
+   *  - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
+   *  - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
+   *     The call failed. If desired this can be used as a trigger to exit the call.
+   *  - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
+   *  - null: EC is configured to never show any waiting for answer state.
+   */
+  public readonly callPickupState$ = this.options.waitForCallPickup
+    ? this.scope.behavior<"unknown" | "ringing" | "timeout" | "success">(
+        concat(
+          concat(
+            // We don't know if the RTC session decides to send a notify event
+            // yet. It will only be known once we sent our own membership and
+            // know we were the first one to join.
+            of("unknown" as const),
+            // Once we get the signal to begin ringing:
+            this.beginRingingForMs$.pipe(
+              take(1),
+              switchMap((lifetime) =>
+                lifetime === 0
+                  ? // If no lifetime, skip the ring state
+                    EMPTY
+                  : // Ring until lifetime ms have passed
+                    timer(lifetime).pipe(
+                      ignoreElements(),
+                      startWith("ringing" as const),
+                    ),
+              ),
+            ),
+            // The notification lifetime has timed out, meaning ringing has
+            // likely stopped on all receiving clients.
+            of("timeout" as const),
+            NEVER,
+          ).pipe(
+            takeUntil(this.someoneElseJoined$.pipe(filter((joined) => joined))),
+          ),
+          of("success" as const),
+        ),
+      )
+    : constant(null);
+
   /**
    * List of MediaItems that we want to display, that are of type ScreenShare
    */

yarn.lock

@@ -10280,7 +10280,7 @@ __metadata:
 
 "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
   version: 37.13.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=3a33c658bbcb8ce8791ec066db899f2571f5c52f"
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=c4c7f945141e142e6f846b243c33c4af97a9a44b"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0"
@@ -10296,7 +10296,7 @@ __metadata:
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:11"
-  checksum: 10c0/1db0d39cfbe4f1c69c8acda0ea7580a4819fc47a7d4bff057382e33e72d9a610f8c03043a6c00bc647dfdc2815aa643c69d25022fb759342a92b77e1841524f1
+  checksum: 10c0/caa4b8a6d924ac36a21773dc2c8be6cb6b658a9feaabccdb24426719c563ac2cfe4778abb86f0889854ae36fc7ba02a6ed39acdbc0b73fdc31ce9a9789e7f36a
   languageName: node
   linkType: hard