Ensure call sound effects are played over the correct sink (#2863)

* Refactor to use AudioContext

* Remove unused audio format.

* Reduce update frequency for volume

* Port to useAudioContext

* Port reactionaudiorenderer to useAudioContext

* Integrate raise hand sound into call event renderer.

* Simplify reaction sounds

* only play one sound per reaction type

* Start to build out tests

* fixup tests / comments

* Fix reaction sound

* remove console line

* Remove another debug line.

* fix lint

* Use testing library click

* lint

* fix a few things

* Change the way we as unknown the mock RTC session.

* Lint

* Fix types for MockRTCSession

* value change should always be set

* Update volume slider description.

* Only load reaction sound effects if enabled.

* cache improvements

* lowercase soundMap

* lint

* move prefetch sounds to fix hot reload

* correct docs

* add a header

* Wording change

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>

src/room/CallEventAudioRenderer.test.tsx

@@ -6,25 +6,30 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { render } from "@testing-library/react";
-import { beforeEach, expect, test } from "vitest";
+import {
+  afterAll,
+  beforeEach,
+  expect,
+  MockedFunction,
+  test,
+  vitest,
+} from "vitest";
 import { MatrixClient } from "matrix-js-sdk/src/client";
-import { ConnectionState, Room } from "livekit-client";
+import { ConnectionState } from "livekit-client";
 import { BehaviorSubject, of } from "rxjs";
 import { afterEach } from "node:test";
-import { act } from "react";
+import { act, ReactNode } from "react";
 import {
   CallMembership,
   type MatrixRTCSession,
 } from "matrix-js-sdk/src/matrixrtc";
+import { RoomMember } from "matrix-js-sdk/src/matrix";
 
-import { soundEffectVolumeSetting } from "../settings/settings";
 import {
-  EmittableMockLivekitRoom,
   mockLivekitRoom,
   mockLocalParticipant,
   mockMatrixRoom,
   mockMatrixRoomMember,
-  mockMediaPlay,
   mockRemoteParticipant,
   mockRtcMembership,
   MockRTCSession,
@@ -35,37 +40,69 @@ import {
   CallEventAudioRenderer,
   MAX_PARTICIPANT_COUNT_FOR_SOUND,
 } from "./CallEventAudioRenderer";
+import { useAudioContext } from "../useAudioContext";
+import { TestReactionsWrapper } from "../utils/testReactions";
+import { prefetchSounds } from "../soundUtils";
 
 const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
 const local = mockMatrixRoomMember(localRtcMember);
 const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
 const alice = mockMatrixRoomMember(aliceRtcMember);
 const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
-const bob = mockMatrixRoomMember(bobRtcMember);
 const localParticipant = mockLocalParticipant({ identity: "" });
 const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
-const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
-const bobParticipant = mockRemoteParticipant({ identity: bobId });
 
-const originalPlayFn = window.HTMLMediaElement.prototype.play;
-
-const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
-const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
-
-beforeEach(() => {
-  soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
-});
+vitest.mock("../useAudioContext");
+vitest.mock("../soundUtils");
 
 afterEach(() => {
-  window.HTMLMediaElement.prototype.play = originalPlayFn;
+  vitest.resetAllMocks();
 });
 
-test("plays a sound when entering a call", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
-  const matrixRoomMembers = new Map(
-    [local, alice, bob].map((p) => [p.userId, p]),
+afterAll(() => {
+  vitest.restoreAllMocks();
+});
+
+let playSound: MockedFunction<
+  NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
+>;
+
+beforeEach(() => {
+  (prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
+    sound: new ArrayBuffer(0),
+  });
+  playSound = vitest.fn();
+  (useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
+    playSound,
+  });
+});
+
+function TestComponent({
+  rtcSession,
+  vm,
+}: {
+  rtcSession: MockRTCSession;
+  vm: CallViewModel;
+}): ReactNode {
+  return (
+    <TestReactionsWrapper
+      rtcSession={rtcSession as unknown as MatrixRTCSession}
+    >
+      <CallEventAudioRenderer vm={vm} />
+    </TestReactionsWrapper>
   );
+}
+
+function getMockEnv(
+  members: RoomMember[],
+  initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
+): {
+  vm: CallViewModel;
+  session: MockRTCSession;
+  remoteRtcMemberships: BehaviorSubject<CallMembership[]>;
+} {
+  const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
   const remoteParticipants = of([aliceParticipant]);
   const liveKitRoom = mockLivekitRoom(
     { localParticipant },
@@ -75,221 +112,90 @@ test("plays a sound when entering a call", () => {
     client: {
       getUserId: () => localRtcMember.sender,
       getDeviceId: () => localRtcMember.deviceId,
+      on: vitest.fn(),
+      off: vitest.fn(),
     } as Partial<MatrixClient> as MatrixClient,
     getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
   });
 
-  const session = new MockRTCSession(matrixRoom, localRtcMember, [
-    aliceRtcMember,
-  ]) as unknown as MatrixRTCSession;
-
-  const vm = new CallViewModel(
-    session,
-    liveKitRoom,
-    {
-      kind: E2eeType.PER_PARTICIPANT,
-    },
-    of(ConnectionState.Connected),
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
+    initialRemoteRtcMemberships,
   );
 
-  render(<CallEventAudioRenderer vm={vm} />);
-  expect(audioIsPlaying).toEqual([
-    // Joining the call
-    enterSound,
-  ]);
-});
-
-test("plays no sound when muted", () => {
-  soundEffectVolumeSetting.setValue(0);
-  const audioIsPlaying: string[] = mockMediaPlay();
-  const matrixRoomMembers = new Map(
-    [local, alice, bob].map((p) => [p.userId, p]),
-  );
-  const remoteParticipants = of([aliceParticipant, bobParticipant]);
-  const liveKitRoom = mockLivekitRoom(
-    { localParticipant },
-    { remoteParticipants },
-  );
-
-  const matrixRoom = mockMatrixRoom({
-    client: {
-      getUserId: () => localRtcMember.sender,
-      getDeviceId: () => localRtcMember.deviceId,
-    } as Partial<MatrixClient> as MatrixClient,
-    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
-  });
-
-  const session = new MockRTCSession(matrixRoom, localRtcMember, [
-    aliceRtcMember,
-  ]) as unknown as MatrixRTCSession;
-
-  const vm = new CallViewModel(
-    session,
-    liveKitRoom,
-    {
-      kind: E2eeType.PER_PARTICIPANT,
-    },
-    of(ConnectionState.Connected),
-  );
-
-  render(<CallEventAudioRenderer vm={vm} />);
-  // Play a sound when joining a call.
-  expect(audioIsPlaying).toHaveLength(0);
-});
-
-test("plays a sound when a user joins", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
-  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map(
-    [aliceParticipant].map((p) => [p.identity, p]),
-  );
-  const liveKitRoom = new EmittableMockLivekitRoom({
-    localParticipant,
-    remoteParticipants,
-  });
-
-  const matrixRoom = mockMatrixRoom({
-    client: {
-      getUserId: () => localRtcMember.sender,
-      getDeviceId: () => localRtcMember.deviceId,
-    } as Partial<MatrixClient> as MatrixClient,
-    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
-  });
-
-  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
-    aliceRtcMember,
-  ]);
-  // we give Bob an RTC session now, but no participant yet
   const session = new MockRTCSession(
     matrixRoom,
     localRtcMember,
-  ).withMemberships(
-    remoteRtcMemberships.asObservable(),
-  ) as unknown as MatrixRTCSession;
+  ).withMemberships(remoteRtcMemberships);
 
   const vm = new CallViewModel(
-    session,
-    liveKitRoom as unknown as Room,
+    session as unknown as MatrixRTCSession,
+    liveKitRoom,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
     of(ConnectionState.Connected),
   );
-  render(<CallEventAudioRenderer vm={vm} />);
+  return { vm, session, remoteRtcMemberships };
+}
+
+/**
+ * We don't want to play a sound when loading the call state
+ * because typically this occurs in two stages. We first join
+ * the call as a local participant and *then* the remote
+ * participants join from our perspective. We don't want to make
+ * a noise every time.
+ */
+test("plays one sound when entering a call", () => {
+  const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
+  render(<TestComponent rtcSession={session} vm={vm} />);
+  // Joining a call usually means remote participants are added later.
+  act(() => {
+    remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
+  });
+  expect(playSound).toHaveBeenCalledOnce();
+});
+
+// TODO: Same test?
+test("plays a sound when a user joins", () => {
+  const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
+  render(<TestComponent rtcSession={session} vm={vm} />);
 
   act(() => {
     remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
   });
   // Play a sound when joining a call.
-  expect(audioIsPlaying).toEqual([
-    // Joining the call
-    enterSound,
-    // Bob joins
-    enterSound,
-  ]);
+  expect(playSound).toBeCalledWith("join");
 });
 
 test("plays a sound when a user leaves", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
-  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map(
-    [aliceParticipant].map((p) => [p.identity, p]),
-  );
-  const liveKitRoom = new EmittableMockLivekitRoom({
-    localParticipant,
-    remoteParticipants,
-  });
-
-  const matrixRoom = mockMatrixRoom({
-    client: {
-      getUserId: () => localRtcMember.sender,
-      getDeviceId: () => localRtcMember.deviceId,
-    } as Partial<MatrixClient> as MatrixClient,
-    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
-  });
-
-  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
-    aliceRtcMember,
-  ]);
-
-  const session = new MockRTCSession(
-    matrixRoom,
-    localRtcMember,
-  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
-
-  const vm = new CallViewModel(
-    session,
-    liveKitRoom as unknown as Room,
-    {
-      kind: E2eeType.PER_PARTICIPANT,
-    },
-    of(ConnectionState.Connected),
-  );
-  render(<CallEventAudioRenderer vm={vm} />);
+  const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
+  render(<TestComponent rtcSession={session} vm={vm} />);
 
   act(() => {
     remoteRtcMemberships.next([]);
   });
-  expect(audioIsPlaying).toEqual([
-    // Joining the call
-    enterSound,
-    // Alice leaves
-    leaveSound,
-  ]);
+  expect(playSound).toBeCalledWith("left");
 });
 
-test("plays no sound when the session member count is larger than the max, until decreased", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
-  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map(
-    [aliceParticipant].map((p) => [p.identity, p]),
-  );
-
+test("plays no sound when the participant list is more than the maximum size", () => {
   const mockRtcMemberships: CallMembership[] = [];
-
   for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
     mockRtcMemberships.push(
       mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
     );
   }
 
-  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
+  const { session, vm, remoteRtcMemberships } = getMockEnv(
+    [local, alice],
     mockRtcMemberships,
   );
 
-  const liveKitRoom = new EmittableMockLivekitRoom({
-    localParticipant,
-    remoteParticipants,
-  });
-
-  const matrixRoom = mockMatrixRoom({
-    client: {
-      getUserId: () => localRtcMember.sender,
-      getDeviceId: () => localRtcMember.deviceId,
-    } as Partial<MatrixClient> as MatrixClient,
-    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
-  });
-
-  const session = new MockRTCSession(
-    matrixRoom,
-    localRtcMember,
-  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
-
-  const vm = new CallViewModel(
-    session,
-    liveKitRoom as unknown as Room,
-    {
-      kind: E2eeType.PER_PARTICIPANT,
-    },
-    of(ConnectionState.Connected),
-  );
-  render(<CallEventAudioRenderer vm={vm} />);
-  expect(audioIsPlaying).toEqual([]);
-  // When the count drops to the max we should play the leave sound
+  render(<TestComponent rtcSession={session} vm={vm} />);
+  expect(playSound).not.toBeCalled();
   act(() => {
     remoteRtcMemberships.next(
       mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
     );
   });
-  expect(audioIsPlaying).toEqual([leaveSound]);
+  expect(playSound).toBeCalledWith("left");
 });

src/room/CallEventAudioRenderer.tsx

@@ -5,47 +5,76 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { ReactNode, useEffect, useRef } from "react";
-import { filter } from "rxjs";
+import { ReactNode, useDeferredValue, useEffect, useMemo } from "react";
+import { filter, interval, throttle } from "rxjs";
 
-import {
-  soundEffectVolumeSetting as effectSoundVolumeSetting,
-  useSetting,
-} from "../settings/settings";
 import { CallViewModel } from "../state/CallViewModel";
-import enterCallSoundMp3 from "../sound/join_call.mp3";
-import enterCallSoundOgg from "../sound/join_call.ogg";
+import joinCallSoundMp3 from "../sound/join_call.mp3";
+import joinCallSoundOgg from "../sound/join_call.ogg";
 import leftCallSoundMp3 from "../sound/left_call.mp3";
 import leftCallSoundOgg from "../sound/left_call.ogg";
+import handSoundOgg from "../sound/raise_hand.ogg?url";
+import handSoundMp3 from "../sound/raise_hand.mp3?url";
+import { useAudioContext } from "../useAudioContext";
+import { prefetchSounds } from "../soundUtils";
+import { useReactions } from "../useReactions";
+import { useLatest } from "../useLatest";
 
 // Do not play any sounds if the participant count has exceeded this
 // number.
 export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
-export const CONCURRENT_AUDIO_CHANNELS = 2;
+export const THROTTLE_SOUND_EFFECT_MS = 500;
+
+const sounds = prefetchSounds({
+  join: {
+    mp3: joinCallSoundMp3,
+    ogg: joinCallSoundOgg,
+  },
+  left: {
+    mp3: leftCallSoundMp3,
+    ogg: leftCallSoundOgg,
+  },
+  raiseHand: {
+    mp3: handSoundMp3,
+    ogg: handSoundOgg,
+  },
+});
 
 export function CallEventAudioRenderer({
   vm,
 }: {
   vm: CallViewModel;
 }): ReactNode {
-  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
-  const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
-  const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
+  const audioEngineCtx = useAudioContext({
+    sounds,
+    latencyHint: "interactive",
+  });
+  const audioEngineRef = useLatest(audioEngineCtx);
+
+  const { raisedHands } = useReactions();
+  const raisedHandCount = useMemo(
+    () => Object.keys(raisedHands).length,
+    [raisedHands],
+  );
+  const previousRaisedHandCount = useDeferredValue(raisedHandCount);
 
   useEffect(() => {
-    if (effectSoundVolume === 0) {
-      return;
+    if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
+      audioEngineRef.current.playSound("raiseHand");
     }
+  }, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
+
+  useEffect(() => {
     const joinSub = vm.memberChanges
       .pipe(
         filter(
           ({ joined, ids }) =>
             ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
         ),
+        throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
       )
-      .subscribe(({ joined }) => {
-        const availablePlayer = callEntered.current.find((v) => v?.paused);
-        void availablePlayer?.play();
+      .subscribe(() => {
+        audioEngineRef.current?.playSound("join");
       });
 
     const leftSub = vm.memberChanges
@@ -54,64 +83,17 @@ export function CallEventAudioRenderer({
           ({ ids, left }) =>
             ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
         ),
+        throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
       )
       .subscribe(() => {
-        const availablePlayer = callLeft.current.find((v) => v?.paused);
-        void availablePlayer?.play();
+        audioEngineRef.current?.playSound("left");
       });
 
     return (): void => {
       joinSub.unsubscribe();
       leftSub.unsubscribe();
     };
-  }, [effectSoundVolume, callEntered, callLeft, vm]);
+  }, [audioEngineRef, vm]);
 
-  // Set volume.
-  useEffect(() => {
-    callEntered.current.forEach((a) => {
-      if (a) {
-        a.volume = effectSoundVolume;
-      }
-    });
-    callLeft.current.forEach((a) => {
-      if (a) {
-        a.volume = effectSoundVolume;
-      }
-    });
-  }, [callEntered, callLeft, effectSoundVolume]);
-
-  // Do not render any audio elements if playback is disabled. Will save
-  // audio file fetches.
-  if (effectSoundVolume === 0) {
-    return null;
-  }
-
-  return (
-    // Will play as soon as it's mounted, which is what we want as this will
-    // play when the call is entered.
-    <>
-      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
-        <audio
-          key={index}
-          ref={(r) => (callEntered.current[index] = r)}
-          preload="auto"
-          hidden
-        >
-          <source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
-          <source src={enterCallSoundMp3} type="audio/mpeg" />
-        </audio>
-      ))}
-      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
-        <audio
-          key={index}
-          ref={(r) => (callLeft.current[index] = r)}
-          preload="auto"
-          hidden
-        >
-          <source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
-          <source src={leftCallSoundMp3} type="audio/mpeg" />
-        </audio>
-      ))}
-    </>
-  );
+  return <></>;
 }

src/room/GroupCallView.tsx

@@ -331,7 +331,7 @@ export const GroupCallView: FC<Props> = ({
         <ActiveCall
           client={client}
           matrixInfo={matrixInfo}
-          rtcSession={rtcSession}
+          rtcSession={rtcSession as unknown as MatrixRTCSession}
           participantCount={participantCount}
           onLeave={onLeave}
           hideHeader={hideHeader}

src/room/InCallView.tsx

@@ -19,7 +19,6 @@ import {
   TouchEvent,
   forwardRef,
   useCallback,
-  useDeferredValue,
   useEffect,
   useMemo,
   useRef,
@@ -81,11 +80,8 @@ import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
 import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
 import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
 import { ReactionsProvider, useReactions } from "../useReactions";
-import handSoundOgg from "../sound/raise_hand.ogg?url";
-import handSoundMp3 from "../sound/raise_hand.mp3?url";
 import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
 import { useSwitchCamera } from "./useSwitchCamera";
-import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
 import { ReactionsOverlay } from "./ReactionsOverlay";
 import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 
@@ -178,14 +174,7 @@ export const InCallView: FC<InCallViewProps> = ({
   connState,
   onShareClick,
 }) => {
-  const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
-  const { supportsReactions, raisedHands, sendReaction, toggleRaisedHand } =
-    useReactions();
-  const raisedHandCount = useMemo(
-    () => Object.keys(raisedHands).length,
-    [raisedHands],
-  );
-  const previousRaisedHandCount = useDeferredValue(raisedHandCount);
+  const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions();
 
   useWakeLock();
 
@@ -335,25 +324,6 @@ export const InCallView: FC<InCallViewProps> = ({
     [vm],
   );
 
-  // Play a sound when the raised hand count increases.
-  const handRaisePlayer = useRef<HTMLAudioElement>(null);
-  useEffect(() => {
-    if (!handRaisePlayer.current) {
-      return;
-    }
-    if (previousRaisedHandCount < raisedHandCount) {
-      handRaisePlayer.current.volume = soundEffectVolume;
-      handRaisePlayer.current.play().catch((ex) => {
-        logger.warn("Failed to play raise hand sound", ex);
-      });
-    }
-  }, [
-    raisedHandCount,
-    handRaisePlayer,
-    previousRaisedHandCount,
-    soundEffectVolume,
-  ]);
-
   useEffect(() => {
     widget?.api.transport
       .send(
@@ -667,10 +637,6 @@ export const InCallView: FC<InCallViewProps> = ({
       <RoomAudioRenderer />
       {renderContent()}
       <CallEventAudioRenderer vm={vm} />
-      <audio ref={handRaisePlayer} preload="auto" hidden>
-        <source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
-        <source src={handSoundMp3} type="audio/mpeg" />
-      </audio>
       <ReactionsAudioRenderer />
       <ReactionsOverlay />
       {footer}

src/room/ReactionAudioRenderer.test.tsx

@@ -6,9 +6,18 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { render } from "@testing-library/react";
-import { afterAll, expect, test } from "vitest";
+import {
+  afterAll,
+  beforeEach,
+  expect,
+  test,
+  vitest,
+  MockedFunction,
+  Mock,
+} from "vitest";
 import { TooltipProvider } from "@vector-im/compound-web";
 import { act, ReactNode } from "react";
+import { afterEach } from "node:test";
 
 import {
   MockRoom,
@@ -16,12 +25,13 @@ import {
   TestReactionsWrapper,
 } from "../utils/testReactions";
 import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
-import { GenericReaction, ReactionSet } from "../reactions";
 import {
   playReactionsSound,
   soundEffectVolumeSetting,
 } from "../settings/settings";
-import { mockMediaPlay } from "../utils/test";
+import { useAudioContext } from "../useAudioContext";
+import { GenericReaction, ReactionSet } from "../reactions";
+import { prefetchSounds } from "../soundUtils";
 
 const memberUserIdAlice = "@alice:example.org";
 const memberUserIdBob = "@bob:example.org";
@@ -50,11 +60,31 @@ function TestComponent({
   );
 }
 
-const originalPlayFn = window.HTMLMediaElement.prototype.play;
-afterAll(() => {
+vitest.mock("../useAudioContext");
+vitest.mock("../soundUtils");
+
+afterEach(() => {
+  vitest.resetAllMocks();
   playReactionsSound.setValue(playReactionsSound.defaultValue);
   soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
-  window.HTMLMediaElement.prototype.play = originalPlayFn;
+});
+
+afterAll(() => {
+  vitest.restoreAllMocks();
+});
+
+let playSound: Mock<
+  NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
+>;
+
+beforeEach(() => {
+  (prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
+    sound: new ArrayBuffer(0),
+  });
+  playSound = vitest.fn();
+  (useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
+    playSound,
+  });
 });
 
 test("preloads all audio elements", () => {
@@ -63,25 +93,11 @@ test("preloads all audio elements", () => {
     new MockRoom(memberUserIdAlice),
     membership,
   );
-  const { container } = render(<TestComponent rtcSession={rtcSession} />);
-  expect(container.getElementsByTagName("audio")).toHaveLength(
-    // All reactions plus the generic sound
-    ReactionSet.filter((r) => r.sound).length + 1,
-  );
-});
-
-test("loads no audio elements when disabled in settings", () => {
-  playReactionsSound.setValue(false);
-  const rtcSession = new MockRTCSession(
-    new MockRoom(memberUserIdAlice),
-    membership,
-  );
-  const { container } = render(<TestComponent rtcSession={rtcSession} />);
-  expect(container.getElementsByTagName("audio")).toHaveLength(0);
+  render(<TestComponent rtcSession={rtcSession} />);
+  expect(prefetchSounds).toHaveBeenCalledOnce();
 });
 
 test("will play an audio sound when there is a reaction", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -97,12 +113,10 @@ test("will play an audio sound when there is a reaction", () => {
   act(() => {
     room.testSendReaction(memberEventAlice, chosenReaction, membership);
   });
-  expect(audioIsPlaying).toHaveLength(1);
-  expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg);
+  expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
 });
 
 test("will play the generic audio sound when there is soundless reaction", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -118,34 +132,10 @@ test("will play the generic audio sound when there is soundless reaction", () =>
   act(() => {
     room.testSendReaction(memberEventAlice, chosenReaction, membership);
   });
-  expect(audioIsPlaying).toHaveLength(1);
-  expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg);
-});
-
-test("will play an audio sound with the correct volume", () => {
-  playReactionsSound.setValue(true);
-  soundEffectVolumeSetting.setValue(0.5);
-  const room = new MockRoom(memberUserIdAlice);
-  const rtcSession = new MockRTCSession(room, membership);
-  const { getByTestId } = render(<TestComponent rtcSession={rtcSession} />);
-
-  // Find the first reaction with a sound effect
-  const chosenReaction = ReactionSet.find((r) => !!r.sound);
-  if (!chosenReaction) {
-    throw Error(
-      "No reactions have sounds configured, this test cannot succeed",
-    );
-  }
-  act(() => {
-    room.testSendReaction(memberEventAlice, chosenReaction, membership);
-  });
-  expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual(
-    0.5,
-  );
+  expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
 });
 
 test("will play multiple audio sounds when there are multiple different reactions", () => {
-  const audioIsPlaying: string[] = mockMediaPlay();
   playReactionsSound.setValue(true);
 
   const room = new MockRoom(memberUserIdAlice);
@@ -164,7 +154,6 @@ test("will play multiple audio sounds when there are multiple different reaction
     room.testSendReaction(memberEventBob, reaction2, membership);
     room.testSendReaction(memberEventCharlie, reaction1, membership);
   });
-  expect(audioIsPlaying).toHaveLength(2);
-  expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg);
-  expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg);
+  expect(playSound).toHaveBeenCalledWith(reaction1.name);
+  expect(playSound).toHaveBeenCalledWith(reaction2.name);
 });

src/room/ReactionAudioRenderer.tsx

@@ -5,70 +5,67 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { ReactNode, useEffect, useRef } from "react";
+import { ReactNode, useDeferredValue, useEffect, useState } from "react";
 
 import { useReactions } from "../useReactions";
-import {
-  playReactionsSound,
-  soundEffectVolumeSetting as effectSoundVolumeSetting,
-  useSetting,
-} from "../settings/settings";
+import { playReactionsSound, useSetting } from "../settings/settings";
 import { GenericReaction, ReactionSet } from "../reactions";
+import { useAudioContext } from "../useAudioContext";
+import { prefetchSounds } from "../soundUtils";
+import { useLatest } from "../useLatest";
+
+const soundMap = Object.fromEntries([
+  ...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
+    v.name,
+    v.sound!,
+  ]),
+  [GenericReaction.name, GenericReaction.sound],
+]);
 
 export function ReactionsAudioRenderer(): ReactNode {
   const { reactions } = useReactions();
   const [shouldPlay] = useSetting(playReactionsSound);
-  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
-  const audioElements = useRef<Record<string, HTMLAudioElement | null>>({});
+  const [soundCache, setSoundCache] = useState<ReturnType<
+    typeof prefetchSounds
+  > | null>(null);
+  const audioEngineCtx = useAudioContext({
+    sounds: soundCache,
+    latencyHint: "interactive",
+  });
+  const audioEngineRef = useLatest(audioEngineCtx);
+  const oldReactions = useDeferredValue(reactions);
 
   useEffect(() => {
-    if (!audioElements.current) {
+    if (!shouldPlay || soundCache) {
       return;
     }
+    // This is fine even if we load the component multiple times,
+    // as the browser's cache should ensure once the media is loaded
+    // once that future fetches come via the cache.
+    setSoundCache(prefetchSounds(soundMap));
+  }, [soundCache, shouldPlay]);
 
-    if (!shouldPlay) {
+  useEffect(() => {
+    if (!shouldPlay || !audioEngineRef.current) {
       return;
     }
+    const oldReactionSet = new Set(
+      Object.values(oldReactions).map((r) => r.name),
+    );
     for (const reactionName of new Set(
       Object.values(reactions).map((r) => r.name),
     )) {
-      const audioElement =
-        audioElements.current[reactionName] ?? audioElements.current.generic;
-      if (audioElement?.paused) {
-        audioElement.volume = effectSoundVolume;
-        void audioElement.play();
+      if (oldReactionSet.has(reactionName)) {
+        // Don't replay old reactions
+        return;
+      }
+      if (soundMap[reactionName]) {
+        audioEngineRef.current.playSound(reactionName);
+      } else {
+        // Fallback sounds.
+        audioEngineRef.current.playSound("generic");
       }
     }
-  }, [audioElements, shouldPlay, reactions, effectSoundVolume]);
-
-  // Do not render any audio elements if playback is disabled. Will save
-  // audio file fetches.
-  if (!shouldPlay) {
-    return null;
-  }
-
-  // NOTE: We load all audio elements ahead of time to allow the cache
-  // to be populated, rather than risk a cache miss and have the audio
-  // be delayed.
-  return (
-    <>
-      {[GenericReaction, ...ReactionSet].map(
-        (r) =>
-          r.sound && (
-            <audio
-              ref={(el) => (audioElements.current[r.name] = el)}
-              data-testid={r.name}
-              key={r.name}
-              preload="auto"
-              hidden
-            >
-              <source src={r.sound.ogg} type="audio/ogg; codecs=vorbis" />
-              {r.sound.mp3 ? (
-                <source src={r.sound.mp3} type="audio/mpeg" />
-              ) : null}
-            </audio>
-          ),
-      )}
-    </>
-  );
+  }, [audioEngineRef, shouldPlay, oldReactions, reactions]);
+  return null;
 }