Add support for playing a sound when the user exits a call. (#2860)

* Refactor to use AudioContext

* Remove unused audio format.

* Reduce update frequency for volume

* Port to useAudioContext

* Port reactionaudiorenderer to useAudioContext

* Integrate raise hand sound into call event renderer.

* Simplify reaction sounds

* only play one sound per reaction type

* Start to build out tests

* fixup tests / comments

* Fix reaction sound

* remove console line

* Remove another debug line.

* fix lint

* Use testing library click

* lint

* Add support for playing a sound when the user exits a call.

* Port GroupCallView to useAudioContext

* Remove debug bits.

* asyncify

* lint

* lint

* lint

* tidy

* Add test for group call view

* Test widget mode too.

* fix ?.

* Format

* Lint

* Lint

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>

src/room/CallEventAudioRenderer.tsx

@@ -25,7 +25,7 @@ import { useLatest } from "../useLatest";
 export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
 export const THROTTLE_SOUND_EFFECT_MS = 500;
 
-const sounds = prefetchSounds({
+export const callEventAudioSounds = prefetchSounds({
   join: {
     mp3: joinCallSoundMp3,
     ogg: joinCallSoundOgg,
@@ -46,7 +46,7 @@ export function CallEventAudioRenderer({
   vm: CallViewModel;
 }): ReactNode {
   const audioEngineCtx = useAudioContext({
-    sounds,
+    sounds: callEventAudioSounds,
     latencyHint: "interactive",
   });
   const audioEngineRef = useLatest(audioEngineCtx);
@@ -60,7 +60,7 @@ export function CallEventAudioRenderer({
 
   useEffect(() => {
     if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
-      audioEngineRef.current.playSound("raiseHand");
+      void audioEngineRef.current.playSound("raiseHand");
     }
   }, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
 
@@ -74,7 +74,7 @@ export function CallEventAudioRenderer({
         throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
       )
       .subscribe(() => {
-        audioEngineRef.current?.playSound("join");
+        void audioEngineRef.current?.playSound("join");
       });
 
     const leftSub = vm.memberChanges
@@ -86,7 +86,7 @@ export function CallEventAudioRenderer({
         throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
       )
       .subscribe(() => {
-        audioEngineRef.current?.playSound("left");
+        void audioEngineRef.current?.playSound("left");
       });
 
     return (): void => {

src/room/GroupCallView.test.tsx

@@ -0,0 +1,153 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { beforeEach, expect, type MockedFunction, test, vitest } from "vitest";
+import { render } from "@testing-library/react";
+import { type MatrixClient } from "matrix-js-sdk/src/client";
+import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
+import { of } from "rxjs";
+import { JoinRule, type RoomState } from "matrix-js-sdk/src/matrix";
+import { Router } from "react-router-dom";
+import { createBrowserHistory } from "history";
+import userEvent from "@testing-library/user-event";
+
+import { type MuteStates } from "./MuteStates";
+import { prefetchSounds } from "../soundUtils";
+import { useAudioContext } from "../useAudioContext";
+import { ActiveCall } from "./InCallView";
+import {
+  mockMatrixRoom,
+  mockMatrixRoomMember,
+  mockRtcMembership,
+  MockRTCSession,
+} from "../utils/test";
+import { GroupCallView } from "./GroupCallView";
+import { leaveRTCSession } from "../rtcSessionHelpers";
+import { type WidgetHelpers } from "../widget";
+import { LazyEventEmitter } from "../LazyEventEmitter";
+
+vitest.mock("../soundUtils");
+vitest.mock("../useAudioContext");
+vitest.mock("./InCallView");
+
+vitest.mock("../rtcSessionHelpers", async (importOriginal) => {
+  // TODO: perhaps there is a more elegant way to manage the type import here?
+  // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+  const orig = await importOriginal<typeof import("../rtcSessionHelpers")>();
+  vitest.spyOn(orig, "leaveRTCSession");
+  return orig;
+});
+
+let playSound: MockedFunction<
+  NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
+>;
+
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
+const carol = mockMatrixRoomMember(localRtcMember);
+const roomMembers = new Map([carol].map((p) => [p.userId, p]));
+
+const roomId = "!foo:bar";
+const soundPromise = Promise.resolve(true);
+
+beforeEach(() => {
+  (prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
+    sound: new ArrayBuffer(0),
+  });
+  playSound = vitest.fn().mockReturnValue(soundPromise);
+  (useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
+    playSound,
+  });
+  // A trivial implementation of Active call to ensure we are testing GroupCallView exclusively here.
+  (ActiveCall as MockedFunction<typeof ActiveCall>).mockImplementation(
+    ({ onLeave }) => {
+      return (
+        <div>
+          <button onClick={() => onLeave()}>Leave</button>
+        </div>
+      );
+    },
+  );
+});
+
+function createGroupCallView(widget: WidgetHelpers | null): {
+  rtcSession: MockRTCSession;
+  getByText: ReturnType<typeof render>["getByText"];
+} {
+  const history = createBrowserHistory();
+  const client = {
+    getUser: () => null,
+    getUserId: () => localRtcMember.sender,
+    getDeviceId: () => localRtcMember.deviceId,
+    getRoom: (rId) => (rId === roomId ? room : null),
+  } as Partial<MatrixClient> as MatrixClient;
+  const room = mockMatrixRoom({
+    client,
+    roomId,
+    getMember: (userId) => roomMembers.get(userId) ?? null,
+    getMxcAvatarUrl: () => null,
+    getCanonicalAlias: () => null,
+    currentState: {
+      getJoinRule: () => JoinRule.Invite,
+    } as Partial<RoomState> as RoomState,
+  });
+  const rtcSession = new MockRTCSession(
+    room,
+    localRtcMember,
+    [],
+  ).withMemberships(of([]));
+  const muteState = {
+    audio: { enabled: false },
+    video: { enabled: false },
+  } as MuteStates;
+  const { getByText } = render(
+    <Router history={history}>
+      <GroupCallView
+        client={client}
+        isPasswordlessUser={false}
+        confineToRoom={false}
+        preload={false}
+        skipLobby={false}
+        hideHeader={true}
+        rtcSession={rtcSession as unknown as MatrixRTCSession}
+        muteStates={muteState}
+        widget={widget}
+      />
+    </Router>,
+  );
+  return {
+    getByText,
+    rtcSession,
+  };
+}
+
+test("will play a leave sound asynchronously in SPA mode", async () => {
+  const user = userEvent.setup();
+  const { getByText, rtcSession } = createGroupCallView(null);
+  const leaveButton = getByText("Leave");
+  await user.click(leaveButton);
+  expect(playSound).toHaveBeenCalledWith("left");
+  expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, undefined);
+  expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
+});
+
+test("will play a leave sound synchronously in widget mode", async () => {
+  const user = userEvent.setup();
+  const widget = {
+    api: {
+      setAlwaysOnScreen: async () => Promise.resolve(true),
+    } as Partial<WidgetHelpers["api"]>,
+    lazyActions: new LazyEventEmitter(),
+  };
+  const { getByText, rtcSession } = createGroupCallView(
+    widget as WidgetHelpers,
+  );
+  const leaveButton = getByText("Leave");
+  await user.click(leaveButton);
+  expect(playSound).toHaveBeenCalledWith("left");
+  expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, soundPromise);
+  expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
+});

src/room/GroupCallView.tsx

@@ -26,7 +26,11 @@ import { Heading, Text } from "@vector-im/compound-web";
 import { useTranslation } from "react-i18next";
 
 import type { IWidgetApiRequest } from "matrix-widget-api";
-import { widget, ElementWidgetActions, type JoinCallData } from "../widget";
+import {
+  ElementWidgetActions,
+  type JoinCallData,
+  type WidgetHelpers,
+} from "../widget";
 import { FullScreenView } from "../FullScreenView";
 import { LobbyView } from "./LobbyView";
 import { type MatrixInfo } from "./VideoPreview";
@@ -51,6 +55,9 @@ import { InviteModal } from "./InviteModal";
 import { useUrlParams } from "../UrlParams";
 import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
+import { useAudioContext } from "../useAudioContext";
+import { callEventAudioSounds } from "./CallEventAudioRenderer";
+import { useLatest } from "../useLatest";
 
 declare global {
   interface Window {
@@ -67,6 +74,7 @@ interface Props {
   hideHeader: boolean;
   rtcSession: MatrixRTCSession;
   muteStates: MuteStates;
+  widget: WidgetHelpers | null;
 }
 
 export const GroupCallView: FC<Props> = ({
@@ -78,10 +86,16 @@ export const GroupCallView: FC<Props> = ({
   hideHeader,
   rtcSession,
   muteStates,
+  widget,
 }) => {
   const memberships = useMatrixRTCSessionMemberships(rtcSession);
   const isJoined = useMatrixRTCSessionJoinState(rtcSession);
-
+  const leaveSoundContext = useLatest(
+    useAudioContext({
+      sounds: callEventAudioSounds,
+      latencyHint: "interactive",
+    }),
+  );
   // This should use `useEffectEvent` (only available in experimental versions)
   useEffect(() => {
     if (memberships.length >= MUTE_PARTICIPANT_COUNT)
@@ -195,14 +209,14 @@ export const GroupCallView: FC<Props> = ({
                 ev.detail.data as unknown as JoinCallData,
               );
               await enterRTCSession(rtcSession, perParticipantE2EE);
-              widget!.api.transport.reply(ev.detail, {});
+              widget.api.transport.reply(ev.detail, {});
             })().catch((e) => {
               logger.error("Error joining RTC session", e);
             });
           };
           widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
           return (): void => {
-            widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
+            widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
           };
         } else {
           // No lobby and no preload: we enter the rtc session right away
@@ -216,7 +230,7 @@ export const GroupCallView: FC<Props> = ({
         void enterRTCSession(rtcSession, perParticipantE2EE);
       }
     }
-  }, [rtcSession, preload, skipLobby, perParticipantE2EE]);
+  }, [widget, rtcSession, preload, skipLobby, perParticipantE2EE]);
 
   const [left, setLeft] = useState(false);
   const [leaveError, setLeaveError] = useState<Error | undefined>(undefined);
@@ -224,12 +238,12 @@ export const GroupCallView: FC<Props> = ({
 
   const onLeave = useCallback(
     (leaveError?: Error): void => {
-      setLeaveError(leaveError);
+      const audioPromise = leaveSoundContext.current?.playSound("left");
-      setLeft(true);
-
       // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent,
       // therefore we want the event to be sent instantly without getting queued/batched.
       const sendInstantly = !!widget;
+      setLeaveError(leaveError);
+      setLeft(true);
       PosthogAnalytics.instance.eventCallEnded.track(
         rtcSession.room.roomId,
         rtcSession.memberships.length,
@@ -237,8 +251,12 @@ export const GroupCallView: FC<Props> = ({
         rtcSession,
       );
 
-      // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
+      leaveRTCSession(
-      leaveRTCSession(rtcSession)
+        rtcSession,
+        // Wait for the sound in widget mode (it's not long)
+        sendInstantly && audioPromise ? audioPromise : undefined,
+      )
+        // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
         .then(() => {
           if (
             !isPasswordlessUser &&
@@ -252,18 +270,25 @@ export const GroupCallView: FC<Props> = ({
           logger.error("Error leaving RTC session", e);
         });
     },
-    [rtcSession, isPasswordlessUser, confineToRoom, history],
+    [
+      widget,
+      rtcSession,
+      isPasswordlessUser,
+      confineToRoom,
+      leaveSoundContext,
+      history,
+    ],
   );
 
   useEffect(() => {
     if (widget && isJoined) {
       // set widget to sticky once joined.
-      widget!.api.setAlwaysOnScreen(true).catch((e) => {
+      widget.api.setAlwaysOnScreen(true).catch((e) => {
         logger.error("Error calling setAlwaysOnScreen(true)", e);
       });
 
       const onHangup = (ev: CustomEvent<IWidgetApiRequest>): void => {
-        widget!.api.transport.reply(ev.detail, {});
+        widget.api.transport.reply(ev.detail, {});
         // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
         leaveRTCSession(rtcSession).catch((e) => {
           logger.error("Failed to leave RTC session", e);
@@ -271,10 +296,10 @@ export const GroupCallView: FC<Props> = ({
       };
       widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup);
       return (): void => {
-        widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
+        widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
       };
     }
-  }, [isJoined, rtcSession]);
+  }, [widget, isJoined, rtcSession]);
 
   const onReconnect = useCallback(() => {
     setLeft(false);
@@ -367,14 +392,17 @@ export const GroupCallView: FC<Props> = ({
       leaveError
     ) {
       return (
-        <CallEndedView
+        <>
-          endedCallId={rtcSession.room.roomId}
+          <CallEndedView
-          client={client}
+            endedCallId={rtcSession.room.roomId}
-          isPasswordlessUser={isPasswordlessUser}
+            client={client}
-          confineToRoom={confineToRoom}
+            isPasswordlessUser={isPasswordlessUser}
-          leaveError={leaveError}
+            confineToRoom={confineToRoom}
-          reconnect={onReconnect}
+            leaveError={leaveError}
-        />
+            reconnect={onReconnect}
+          />
+          ;
+        </>
       );
     } else {
       // If the user is a regular user, we'll have sent them back to the homepage,

src/room/ReactionAudioRenderer.tsx

@@ -60,10 +60,10 @@ export function ReactionsAudioRenderer(): ReactNode {
         return;
       }
       if (soundMap[reactionName]) {
-        audioEngineRef.current.playSound(reactionName);
+        void audioEngineRef.current.playSound(reactionName);
       } else {
         // Fallback sounds.
-        audioEngineRef.current.playSound("generic");
+        void audioEngineRef.current.playSound("generic");
       }
     }
   }, [audioEngineRef, shouldPlay, oldReactions, reactions]);

src/room/RoomPage.tsx

@@ -98,6 +98,7 @@ export const RoomPage: FC = () => {
       case "loaded":
         return (
           <GroupCallView
+            widget={widget}
             client={client!}
             rtcSession={groupCallState.rtcSession}
             isPasswordlessUser={passwordlessUser}

src/rtcSessionHelpers.ts

@@ -120,6 +120,7 @@ export async function enterRTCSession(
 
 const widgetPostHangupProcedure = async (
   widget: WidgetHelpers,
+  promiseBeforeHangup?: Promise<unknown>,
 ): Promise<void> => {
   // we need to wait until the callEnded event is tracked on posthog.
   // Otherwise the iFrame gets killed before the callEnded event got tracked.
@@ -132,6 +133,8 @@ const widgetPostHangupProcedure = async (
     logger.error("Failed to set call widget `alwaysOnScreen` to false", e);
   }
 
+  // Wait for any last bits before hanging up.
+  await promiseBeforeHangup;
   // We send the hangup event after the memberships have been updated
   // calling leaveRTCSession.
   // We need to wait because this makes the client hosting this widget killing the IFrame.
@@ -140,9 +143,12 @@ const widgetPostHangupProcedure = async (
 
 export async function leaveRTCSession(
   rtcSession: MatrixRTCSession,
+  promiseBeforeHangup?: Promise<unknown>,
 ): Promise<void> {
   await rtcSession.leaveRoomSession();
   if (widget) {
-    await widgetPostHangupProcedure(widget);
+    await widgetPostHangupProcedure(widget, promiseBeforeHangup);
+  } else {
+    await promiseBeforeHangup;
   }
 }

src/useAudioContext.test.tsx

@@ -29,9 +29,11 @@ const TestComponent: FC = () => {
   }
   return (
     <>
-      <button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
+      <button onClick={() => void audioCtx.playSound("aSound")}>
+        Valid sound
+      </button>
       {/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
-      <button onClick={() => audioCtx.playSound("not-valid" as any)}>
+      <button onClick={() => void audioCtx.playSound("not-valid" as any)}>
         Invalid sound
       </button>
     </>
@@ -61,6 +63,7 @@ class MockAudioContext {
     vitest.mocked({
       connect: (v: unknown) => v,
       start: () => {},
+      addEventListener: (_name: string, cb: () => void) => cb(),
     }),
   );
   public createGain = vitest.fn().mockReturnValue(this.gain);

src/useAudioContext.tsx

@@ -22,18 +22,21 @@ import { type PrefetchedSounds } from "./soundUtils";
  * @param volume The volume to play at.
  * @param ctx The context to play through.
  * @param buffer The buffer to play.
+ * @returns A promise that resolves when the sound has finished playing.
  */
-function playSound(
+async function playSound(
   ctx: AudioContext,
   buffer: AudioBuffer,
   volume: number,
-): void {
+): Promise<void> {
   const gain = ctx.createGain();
   gain.gain.setValueAtTime(volume, 0);
   const src = ctx.createBufferSource();
   src.buffer = buffer;
   src.connect(gain).connect(ctx.destination);
+  const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
   src.start();
+  return p;
 }
 
 interface Props<S extends string> {
@@ -47,7 +50,7 @@ interface Props<S extends string> {
 }
 
 interface UseAudioContext<S> {
-  playSound(soundName: S): void;
+  playSound(soundName: S): Promise<void>;
 }
 
 /**
@@ -113,7 +116,7 @@ export function useAudioContext<S extends string>(
     return null;
   }
   return {
-    playSound: (name): void => {
+    playSound: async (name): Promise<void> => {
       if (!audioBuffers[name]) {
         logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
         return;

src/utils/test.ts

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 import { map, type Observable, of, type SchedulerLike } from "rxjs";
 import { type RunHelpers, TestScheduler } from "rxjs/testing";
-import { expect, vi } from "vitest";
+import { expect, vi, vitest } from "vitest";
 import {
   type RoomMember,
   type Room as MatrixRoom,
@@ -258,6 +258,12 @@ export class MockRTCSession extends TypedEventEmitter<
   MatrixRTCSessionEvent,
   MatrixRTCSessionEventHandlerMap
 > {
+  public readonly statistics = {
+    counters: {},
+  };
+
+  public leaveRoomSession = vitest.fn().mockResolvedValue(undefined);
+
   public constructor(
     public readonly room: Room,
     private localMembership: CallMembership,
@@ -266,6 +272,10 @@ export class MockRTCSession extends TypedEventEmitter<
     super();
   }
 
+  public isJoined(): true {
+    return true;
+  }
+
   public withMemberships(
     rtcMembers: Observable<Partial<CallMembership>[]>,
   ): MockRTCSession {