Add custom audio renderer for iPhone earpiece and only render joined participants (#3249)

* Add custom audio renderer to only render joined participants & add ios earpice workaround

fix left right to match chromium + safari
(firefox is swapped)

earpice as setting

Simpler code and documentation
The doc explains, what this class actually does and why it is so complicated.

Signed-off-by: Timo K <toger5@hotmail.de>

use only one audioContext, remove (non working) standby fallback

* Add tests

* use optional audio context and effect to initiate it + review

locales/en/app.json

@@ -61,6 +61,7 @@
     "video": "Video"
   },
   "developer_mode": {
+    "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms",
     "crypto_version": "Crypto version: {{version}}",
     "debug_tile_layout_label": "Debug tile layout",
     "device_id": "Device ID: {{id}}",
@@ -174,6 +175,7 @@
       "camera_numbered": "Camera {{n}}",
       "default": "Default",
       "default_named": "Default <2>({{name}})</2>",
+      "earpiece": "Earpiece",
       "microphone": "Microphone",
       "microphone_numbered": "Microphone {{n}}",
       "speaker": "Speaker",

playwright/access.spec.ts

@@ -49,7 +49,7 @@ test("Sign up a new account, then login, then logout", async ({ browser }) => {
 
   // logout
   await returningUserPage.getByTestId("usermenu_open").click();
-  await returningUserPage.locator('[data-test-id="usermenu_logout"]').click();
+  await returningUserPage.locator('[data-testid="usermenu_logout"]').click();
 
   await expect(
     returningUserPage.getByRole("link", { name: "Log In" }),

src/UserMenu.tsx

@@ -119,7 +119,7 @@ export const UserMenu: FC<Props> = ({
           key={key}
           Icon={Icon}
           label={label}
-          data-test-id={dataTestid}
+          data-testid={dataTestid}
           onSelect={() => onAction(key)}
         />
       ))}

src/livekit/MatrixAudioRenderer.test.tsx

@@ -0,0 +1,104 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { render } from "@testing-library/react";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import {
+  getTrackReferenceId,
+  type TrackReference,
+} from "@livekit/components-core";
+import { type RemoteAudioTrack } from "livekit-client";
+import { type ReactNode } from "react";
+import { useTracks } from "@livekit/components-react";
+
+import { testAudioContext } from "../useAudioContext.test";
+import * as MediaDevicesContext from "./MediaDevicesContext";
+import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
+import { mockTrack } from "../utils/test";
+
+export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
+
+beforeEach(() => {
+  vi.stubGlobal("AudioContext", TestAudioContextConstructor);
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  vi.clearAllMocks();
+});
+
+vi.mock("@livekit/components-react", async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
+      return (
+        <audio data-testid={"audio"}>
+          {getTrackReferenceId(props.trackRef)}
+        </audio>
+      );
+    },
+    useTracks: vi.fn(),
+  };
+});
+
+const tracks = [mockTrack("test:123")];
+vi.mocked(useTracks).mockReturnValue(tracks);
+
+it("should render for member", () => {
+  const { container, queryAllByTestId } = render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  expect(container).toBeTruthy();
+  expect(queryAllByTestId("audio")).toHaveLength(1);
+});
+it("should not render without member", () => {
+  const { container, queryAllByTestId } = render(
+    <MatrixAudioRenderer
+      members={[{ sender: "othermember", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  expect(container).toBeTruthy();
+  expect(queryAllByTestId("audio")).toHaveLength(0);
+});
+
+it("should not setup audioContext gain and pan if there is no need to.", () => {
+  render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
+
+  expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
+  expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined);
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1);
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]);
+
+  expect(testAudioContext.gain.gain.value).toEqual(1);
+  expect(testAudioContext.pan.pan.value).toEqual(0);
+});
+it("should setup audioContext gain and pan", () => {
+  vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
+    pan: 1,
+    volume: 0.1,
+  });
+  render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+
+  const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
+  expect(audioTrack.setAudioContext).toHaveBeenCalled();
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled();
+
+  expect(testAudioContext.gain.gain.value).toEqual(0.1);
+  expect(testAudioContext.pan.pan.value).toEqual(1);
+});

src/livekit/MatrixAudioRenderer.tsx

@@ -0,0 +1,212 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { getTrackReferenceId } from "@livekit/components-core";
+import { type RemoteAudioTrack, Track } from "livekit-client";
+import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
+import {
+  useTracks,
+  AudioTrack,
+  type AudioTrackProps,
+} from "@livekit/components-react";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { useEarpieceAudioConfig } from "./MediaDevicesContext";
+import { useReactiveState } from "../useReactiveState";
+
+export interface MatrixAudioRendererProps {
+  /**
+   * The list of participants to render audio for.
+   * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
+   * that are not expected to be in the rtc session.
+   */
+  members: CallMembership[];
+  /**
+   * If set to `true`, mutes all audio tracks rendered by the component.
+   * @remarks
+   * If set to `true`, the server will stop sending audio track data to the client.
+   */
+  muted?: boolean;
+}
+
+/**
+ * The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
+ * It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
+ *
+ * It also takes care of the earpiece audio configuration for iOS devices.
+ * This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
+ * @example
+ * ```tsx
+ * <LiveKitRoom>
+ *   <MatrixAudioRenderer />
+ * </LiveKitRoom>
+ * ```
+ * @public
+ */
+export function MatrixAudioRenderer({
+  members,
+  muted,
+}: MatrixAudioRendererProps): ReactNode {
+  const validIdentities = useMemo(
+    () =>
+      new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
+    [members],
+  );
+
+  const loggedInvalidIdentities = useRef(new Set<string>());
+  /**
+   * Log an invalid livekit track identity.
+   * A invalid identity is one that does not match any of the matrix rtc members.
+   *
+   * @param identity The identity of the track that is invalid
+   * @param validIdentities The list of valid identities
+   */
+  const logInvalid = (identity: string, validIdentities: Set<string>): void => {
+    if (loggedInvalidIdentities.current.has(identity)) return;
+    logger.warn(
+      `Audio track ${identity} has no matching matrix call member`,
+      `current members: ${Array.from(validIdentities.values())}`,
+      `track will not get rendered`,
+    );
+    loggedInvalidIdentities.current.add(identity);
+  };
+
+  const tracks = useTracks(
+    [
+      Track.Source.Microphone,
+      Track.Source.ScreenShareAudio,
+      Track.Source.Unknown,
+    ],
+    {
+      updateOnlyOn: [],
+      onlySubscribed: true,
+    },
+  ).filter((ref) => {
+    const isValid = validIdentities?.has(ref.participant.identity);
+    if (!isValid && !ref.participant.isLocal)
+      logInvalid(ref.participant.identity, validIdentities);
+    return (
+      !ref.participant.isLocal &&
+      ref.publication.kind === Track.Kind.Audio &&
+      isValid
+    );
+  });
+
+  // This component is also (in addition to the "only play audio for connected members" logic above)
+  // responsible for mimicking earpiece audio on iPhones.
+  // The Safari audio devices enumeration does not expose an earpiece audio device.
+  // We alternatively use the audioContext pan node to only use one of the stereo channels.
+
+  // This component does get additionally complicated because of a Safari bug.
+  // (see: https://bugs.webkit.org/show_bug.cgi?id=251532
+  // and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
+  // and https://bugs.webkit.org/show_bug.cgi?id=231105)
+  //
+  // AudioContext gets stopped if the webview gets moved into the background.
+  // Once the phone is in standby audio playback will stop.
+  // So we can only use the pan trick only works is the phone is not in standby.
+  // If earpiece mode is not used we do not use audioContext to allow standby playback.
+  // shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
+
+  const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
+  const shouldUseAudioContext = stereoPan !== 0;
+
+  // initialize the potentially used audio context.
+  const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
+    undefined,
+  );
+  useEffect(() => {
+    const ctx = new AudioContext();
+    setAudioContext(ctx);
+    return (): void => {
+      void ctx.close();
+    };
+  }, []);
+  const audioNodes = useMemo(
+    () => ({
+      gain: audioContext?.createGain(),
+      pan: audioContext?.createStereoPanner(),
+    }),
+    [audioContext],
+  );
+
+  // Simple effects to update the gain and pan node based on the props
+  useEffect(() => {
+    if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
+  }, [audioNodes.pan, stereoPan]);
+  useEffect(() => {
+    if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
+  }, [audioNodes.gain, volumeFactor]);
+
+  return (
+    // We add all audio elements into one <div> for the browser developer tool experience/tidyness.
+    <div style={{ display: "none" }}>
+      {tracks.map((trackRef) => (
+        <AudioTrackWithAudioNodes
+          key={getTrackReferenceId(trackRef)}
+          trackRef={trackRef}
+          muted={muted}
+          audioContext={shouldUseAudioContext ? audioContext : undefined}
+          audioNodes={audioNodes}
+        />
+      ))}
+    </div>
+  );
+}
+
+interface StereoPanAudioTrackProps {
+  muted?: boolean;
+  audioContext?: AudioContext;
+  audioNodes: {
+    gain?: GainNode;
+    pan?: StereoPannerNode;
+  };
+}
+
+/**
+ * This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
+ * It main purpose is to remount the AudioTrack component when switching from
+ * audiooContext to normal audio playback.
+ * As of now the AudioTrack component does not support adding audio nodes while being mounted.
+ * @param param0
+ * @returns
+ */
+function AudioTrackWithAudioNodes({
+  trackRef,
+  muted,
+  audioContext,
+  audioNodes,
+  ...props
+}: StereoPanAudioTrackProps &
+  AudioTrackProps &
+  React.RefAttributes<HTMLAudioElement>): ReactNode {
+  // This is used to unmount/remount the AudioTrack component.
+  // Mounting needs to happen after the audioContext is set.
+  // (adding the audio context when already mounted did not work outside strict mode)
+  const [trackReady, setTrackReady] = useReactiveState(
+    () => false,
+    // We only want the track to reset once both (audioNodes and audioContext) are set.
+    // for unsetting the audioContext its enough if one of the the is undefined.
+    [audioContext && audioNodes],
+  );
+
+  useEffect(() => {
+    if (!trackRef || trackReady) return;
+    const track = trackRef.publication.track as RemoteAudioTrack;
+    const useContext = audioContext && audioNodes.gain && audioNodes.pan;
+    track.setAudioContext(useContext ? audioContext : undefined);
+    track.setWebAudioPlugins(
+      useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
+    );
+    setTrackReady(true);
+  }, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
+
+  return (
+    trackReady && <AudioTrack trackRef={trackRef} muted={muted} {...props} />
+  );
+}

src/livekit/MediaDevicesContext.tsx

@@ -26,12 +26,16 @@ import {
   audioInput as audioInputSetting,
   audioOutput as audioOutputSetting,
   videoInput as videoInputSetting,
+  alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
   type Setting,
 } from "../settings/settings";
 
+export const EARPIECE_CONFIG_ID = "earpiece-id";
+
 export type DeviceLabel =
   | { type: "name"; name: string }
   | { type: "number"; number: number }
+  | { type: "earpiece" }
   | { type: "default"; name: string | null };
 
 export interface MediaDevice {
@@ -40,6 +44,11 @@ export interface MediaDevice {
    */
   available: Map<string, DeviceLabel>;
   selectedId: string | undefined;
+  /**
+   * An additional device configuration that makes us use only one channel of the
+   * output device and a reduced volume.
+   */
+  useAsEarpiece: boolean | undefined;
   /**
    * The group ID of the selected device.
    */
@@ -65,6 +74,7 @@ function useMediaDevice(
 ): MediaDevice {
   // Make sure we don't needlessly reset to a device observer without names,
   // once permissions are already given
+  const [alwaysShowIphoneEarpice] = useSetting(alwaysShowIphoneEarpieceSetting);
   const hasRequestedPermissions = useRef(false);
   const requestPermissions = usingNames || hasRequestedPermissions.current;
   hasRequestedPermissions.current ||= usingNames;
@@ -102,27 +112,39 @@ function useMediaDevice(
             // Create a virtual default audio output for browsers that don't have one.
             // Its device ID must be the empty string because that's what setSinkId
             // recognizes.
+            // We also create this if we do not have any available devices, so that
+            // we can use the default or the earpiece.
+            const showEarpiece =
+              navigator.userAgent.match("iPhone") || alwaysShowIphoneEarpice;
             if (
               kind === "audiooutput" &&
-              available.size &&
               !available.has("") &&
-              !available.has("default")
+              !available.has("default") &&
+              (available.size || showEarpiece)
             )
               available = new Map([
                 ["", { type: "default", name: availableRaw[0]?.label || null }],
                 ...available,
               ]);
+            if (kind === "audiooutput" && showEarpiece)
+              // On IPhones we have to create a virtual earpiece device, because
+              // the earpiece is not available as a device ID.
+              available = new Map([
+                ...available,
+                [EARPIECE_CONFIG_ID, { type: "earpiece" }],
+              ]);
             // Note: creating virtual default input devices would be another problem
             // entirely, because requesting a media stream from deviceId "" won't
             // automatically track the default device.
             return available;
           }),
         ),
-      [kind, deviceObserver$],
+      [alwaysShowIphoneEarpice, deviceObserver$, kind],
     ),
   );
 
-  const [preferredId, select] = useSetting(setting);
+  const [preferredId, setPreferredId] = useSetting(setting);
+  const [asEarpice, setAsEarpiece] = useState(false);
   const selectedId = useMemo(() => {
     if (available.size) {
       // If the preferred device is available, use it. Or if every available
@@ -138,6 +160,7 @@ function useMediaDevice(
     }
     return undefined;
   }, [available, preferredId]);
+
   const selectedGroupId = useObservableEagerState(
     useMemo(
       () =>
@@ -151,14 +174,27 @@ function useMediaDevice(
     ),
   );
 
+  const select = useCallback(
+    (id: string) => {
+      if (id === EARPIECE_CONFIG_ID) {
+        setAsEarpiece(true);
+      } else {
+        setAsEarpiece(false);
+        setPreferredId(id);
+      }
+    },
+    [setPreferredId],
+  );
+
   return useMemo(
     () => ({
       available,
       selectedId,
+      useAsEarpiece: asEarpice,
       selectedGroupId,
       select,
     }),
-    [available, selectedId, selectedGroupId, select],
+    [available, selectedId, asEarpice, selectedGroupId, select],
   );
 }
 
@@ -167,6 +203,7 @@ export const deviceStub: MediaDevice = {
   selectedId: undefined,
   selectedGroupId: undefined,
   select: () => {},
+  useAsEarpiece: false,
 };
 export const devicesStub: MediaDevices = {
   audioInput: deviceStub,
@@ -255,3 +292,30 @@ export const useMediaDeviceNames = (
       return context.stopUsingDeviceNames;
     }
   }, [context, enabled]);
+
+/**
+ * A convenience hook to get the audio node configuration for the earpiece.
+ * It will check the `useAsEarpiece` of the `audioOutput` device and return
+ * the appropriate pan and volume values.
+ *
+ * @returns pan and volume values for the earpiece audio node configuration.
+ */
+export const useEarpieceAudioConfig = (): {
+  pan: number;
+  volume: number;
+} => {
+  const { audioOutput } = useMediaDevices();
+  // We use only the right speaker (pan = 1) for the earpiece.
+  // This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone)
+  const pan = useMemo(
+    () => (audioOutput.useAsEarpiece ? 1 : 0),
+    [audioOutput.useAsEarpiece],
+  );
+  // We also do lower the volume by a factor of 10 to optimize for the usecase where
+  // a user is holding the phone to their ear.
+  const volume = useMemo(
+    () => (audioOutput.useAsEarpiece ? 0.1 : 1),
+    [audioOutput.useAsEarpiece],
+  );
+  return { pan, volume };
+};

src/room/InCallView.test.tsx

@@ -21,11 +21,7 @@ import { ConnectionState, type LocalParticipant } from "livekit-client";
 import { of } from "rxjs";
 import { BrowserRouter } from "react-router-dom";
 import { TooltipProvider } from "@vector-im/compound-web";
-import {
+import { RoomContext, useLocalParticipant } from "@livekit/components-react";
-  RoomAudioRenderer,
-  RoomContext,
-  useLocalParticipant,
-} from "@livekit/components-react";
 import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
 
 import { type MuteStates } from "./MuteStates";
@@ -48,6 +44,7 @@ import {
 } from "../settings/settings";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
+import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
 
 // vi.hoisted(() => {
 //   localStorage = {} as unknown as Storage;
@@ -65,6 +62,7 @@ vi.mock("../tile/GridTile");
 vi.mock("../tile/SpotlightTile");
 vi.mock("@livekit/components-react");
 vi.mock("../e2ee/sharedKeyManagement");
+vi.mock("../livekit/MatrixAudioRenderer");
 vi.mock("react-use-measure", () => ({
   default: (): [() => void, object] => [(): void => {}, {}],
 }));
@@ -81,13 +79,15 @@ const roomMembers = new Map([carol].map((p) => [p.userId, p]));
 
 const roomId = "!foo:bar";
 let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
+
 beforeEach(() => {
   vi.clearAllMocks();
-  // RoomAudioRenderer is tested separately.
+
+  // MatrixAudioRenderer is tested separately.
   (
-    RoomAudioRenderer as MockedFunction<typeof RoomAudioRenderer>
+    MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
   ).mockImplementation((_props) => {
-    return <div>mocked: RoomAudioRenderer</div>;
+    return <div>mocked: MatrixAudioRenderer</div>;
   });
   (
     useLocalParticipant as MockedFunction<typeof useLocalParticipant>
@@ -98,7 +98,6 @@ beforeEach(() => {
         localParticipant: localRtcMember as unknown as LocalParticipant,
       }) as unknown as ReturnType<typeof useLocalParticipant>,
   );
-
   useRoomEncryptionSystemMock =
     useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
   useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });

src/room/InCallView.tsx

@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
+import { RoomContext, useLocalParticipant } from "@livekit/components-react";
-  RoomAudioRenderer,
-  RoomContext,
-  useLocalParticipant,
-} from "@livekit/components-react";
 import { Text } from "@vector-im/compound-web";
 import { ConnectionState, type Room } from "livekit-client";
 import { type MatrixClient } from "matrix-js-sdk";
@@ -107,6 +103,7 @@ import {
 import { ReactionsReader } from "../reactions/ReactionsReader";
 import { ConnectionLostError } from "../utils/errors.ts";
 import { useTypedEventEmitter } from "../useEvents.ts";
+import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -726,7 +723,10 @@ export const InCallView: FC<InCallViewProps> = ({
           </Text>
         )
       }
-      <RoomAudioRenderer muted={muteAllAudio} />
+      <MatrixAudioRenderer
+        members={rtcSession.memberships}
+        muted={muteAllAudio}
+      />
       {renderContent()}
       <CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
       <ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />

src/room/MuteStates.test.tsx

@@ -79,6 +79,7 @@ function mockDevices(available: Map<string, DeviceLabel>): MediaDevice {
     selectedId: "",
     selectedGroupId: "",
     select: (): void => {},
+    useAsEarpiece: false,
   };
 }
 

src/room/__snapshots__/InCallView.test.tsx.snap

@@ -9,7 +9,7 @@ exports[`InCallView > rendering > renders 1`] = `
       class="header filler"
     />
     <div>
-      mocked: RoomAudioRenderer
+      mocked: MatrixAudioRenderer
     </div>
     <div
       class="scrollingGrid grid"

src/settings/DeveloperSettingsTab.tsx

@@ -18,6 +18,7 @@ import {
   useNewMembershipManager as useNewMembershipManagerSetting,
   useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
   muteAllAudio as muteAllAudioSetting,
+  alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk";
 import type { Room as LivekitRoom } from "livekit-client";
@@ -46,6 +47,9 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
     useNewMembershipManagerSetting,
   );
 
+  const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
+    alwaysShowIphoneEarpieceSetting,
+  );
   const [
     useExperimentalToDeviceTransport,
     setUseExperimentalToDeviceTransport,
@@ -192,6 +196,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
             [setMuteAllAudio],
           )}
         />
+      </FieldRow>{" "}
+      <FieldRow>
+        <InputField
+          id="alwaysShowIphoneEarpiece"
+          type="checkbox"
+          label={t("developer_mode.always_show_iphone_earpiece")}
+          checked={alwaysShowIphoneEarpiece}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setAlwaysShowIphoneEarpiece(event.target.checked);
+            },
+            [setAlwaysShowIphoneEarpiece],
+          )}
+        />{" "}
       </FieldRow>
       {livekitRoom ? (
         <>

src/settings/DeviceSelection.tsx

@@ -22,17 +22,20 @@ import {
 } from "@vector-im/compound-web";
 import { Trans, useTranslation } from "react-i18next";
 
-import { type MediaDevice } from "../livekit/MediaDevicesContext";
+import {
+  EARPIECE_CONFIG_ID,
+  type MediaDevice,
+} from "../livekit/MediaDevicesContext";
 import styles from "./DeviceSelection.module.css";
 
 interface Props {
-  devices: MediaDevice;
+  device: MediaDevice;
   title: string;
   numberedLabel: (number: number) => string;
 }
 
 export const DeviceSelection: FC<Props> = ({
-  devices,
+  device,
   title,
   numberedLabel,
 }) => {
@@ -40,12 +43,13 @@ export const DeviceSelection: FC<Props> = ({
   const groupId = useId();
   const onChange = useCallback(
     (e: ChangeEvent<HTMLInputElement>) => {
-      devices.select(e.target.value);
+      device.select(e.target.value);
     },
-    [devices],
+    [device],
   );
 
-  if (devices.available.size == 0) return null;
+  // There is no need to show the menu if there is no choice that can be made.
+  if (device.available.size <= 1) return null;
 
   return (
     <div className={styles.selection}>
@@ -60,7 +64,7 @@ export const DeviceSelection: FC<Props> = ({
       </Heading>
       <Separator className={styles.separator} />
       <div className={styles.options}>
-        {[...devices.available].map(([id, label]) => {
+        {[...device.available].map(([id, label]) => {
           let labelText: ReactNode;
           switch (label.type) {
             case "name":
@@ -85,6 +89,16 @@ export const DeviceSelection: FC<Props> = ({
                   </Trans>
                 );
               break;
+            case "earpiece":
+              labelText = t("settings.devices.earpiece");
+              break;
+          }
+
+          let isSelected = false;
+          if (device.useAsEarpiece) {
+            isSelected = id === EARPIECE_CONFIG_ID;
+          } else {
+            isSelected = id === device.selectedId;
           }
 
           return (
@@ -93,7 +107,7 @@ export const DeviceSelection: FC<Props> = ({
               name={groupId}
               control={
                 <RadioControl
-                  checked={id === devices.selectedId}
+                  checked={isSelected}
                   onChange={onChange}
                   value={id}
                 />

src/settings/SettingsModal.tsx

@@ -98,7 +98,6 @@ export const SettingsModal: FC<Props> = ({
   useMediaDeviceNames(devices, open);
   const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
   const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
-
   const [showDeveloperSettingsTab] = useSetting(developerMode);
 
   const { available: isRageshakeAvailable } = useSubmitRageshake();
@@ -110,17 +109,18 @@ export const SettingsModal: FC<Props> = ({
       <>
         <Form>
           <DeviceSelection
-            devices={devices.audioInput}
+            device={devices.audioInput}
             title={t("settings.devices.microphone")}
             numberedLabel={(n) =>
               t("settings.devices.microphone_numbered", { n })
             }
           />
           <DeviceSelection
-            devices={devices.audioOutput}
+            device={devices.audioOutput}
             title={t("settings.devices.speaker")}
             numberedLabel={(n) => t("settings.devices.speaker_numbered", { n })}
           />
+
           <div className={styles.volumeSlider}>
             <label>{t("settings.audio_tab.effect_volume_label")}</label>
             <p>{t("settings.audio_tab.effect_volume_description")}</p>
@@ -146,7 +146,7 @@ export const SettingsModal: FC<Props> = ({
       <>
         <Form>
           <DeviceSelection
-            devices={devices.videoInput}
+            device={devices.videoInput}
             title={t("settings.devices.camera")}
             numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
           />

src/settings/settings.ts

@@ -44,6 +44,9 @@ export class Setting<T> {
     this._value$.next(value);
     localStorage.setItem(this.key, JSON.stringify(value));
   };
+  public readonly getValue = (): T => {
+    return this._value$.getValue();
+  };
 }
 
 /**
@@ -128,3 +131,8 @@ export const useExperimentalToDeviceTransport = new Setting<boolean>(
 export const muteAllAudio = new Setting<boolean>("mute-all-audio", false);
 
 export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);
+
+export const alwaysShowIphoneEarpiece = new Setting<boolean>(
+  "always-show-iphone-earpice",
+  false,
+);

src/useAudioContext.test.tsx

@@ -5,10 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { expect, test, vitest, afterEach } from "vitest";
+import { expect, vi, afterEach, beforeEach, test } from "vitest";
 import { type FC } from "react";
 import { render } from "@testing-library/react";
-import userEvent from "@testing-library/user-event";
+import userEvent, { type UserEvent } from "@testing-library/user-event";
 
 import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
 import { useAudioContext } from "./useAudioContext";
@@ -39,61 +39,73 @@ const TestComponent: FC = () => {
   );
 };
 
-class MockAudioContext {
+const gainNode = vi.mocked(
-  public static testContext: MockAudioContext;
+  {
-
+    connect: (node: AudioNode) => node,
-  public constructor() {
+    gain: {
-    MockAudioContext.testContext = this;
+      setValueAtTime: vi.fn(),
-  }
+      value: 1,
-
-  public gain = vitest.mocked(
-    {
-      connect: () => {},
-      gain: {
-        setValueAtTime: vitest.fn(),
-      },
     },
-    true,
+  },
-  );
+  true,
-
+);
-  public setSinkId = vitest.fn().mockResolvedValue(undefined);
+const panNode = vi.mocked(
-  public decodeAudioData = vitest.fn().mockReturnValue(1);
+  {
-  public createBufferSource = vitest.fn().mockReturnValue(
+    connect: (node: AudioNode) => node,
-    vitest.mocked({
+    pan: {
+      setValueAtTime: vi.fn(),
+      value: 0,
+    },
+  },
+  true,
+);
+/**
+ * A shared audio context test instance.
+ * It can also be used to mock the `AudioContext` constructor in tests:
+ * `vi.stubGlobal("AudioContext", () => testAudioContext);`
+ */
+export const testAudioContext = {
+  gain: gainNode,
+  pan: panNode,
+  setSinkId: vi.fn().mockResolvedValue(undefined),
+  decodeAudioData: vi.fn().mockReturnValue(1),
+  createBufferSource: vi.fn().mockReturnValue(
+    vi.mocked({
       connect: (v: unknown) => v,
       start: () => {},
       addEventListener: (_name: string, cb: () => void) => cb(),
     }),
-  );
+  ),
-  public createGain = vitest.fn().mockReturnValue(this.gain);
+  createGain: vi.fn().mockReturnValue(gainNode),
-  public close = vitest.fn().mockResolvedValue(undefined);
+  createStereoPanner: vi.fn().mockReturnValue(panNode),
-}
+  close: vi.fn().mockResolvedValue(undefined),
+};
+export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
+
+let user: UserEvent;
+beforeEach(() => {
+  vi.stubGlobal("AudioContext", TestAudioContextConstructor);
+  user = userEvent.setup();
+});
 
 afterEach(() => {
-  vitest.unstubAllGlobals();
+  vi.unstubAllGlobals();
+  vi.clearAllMocks();
 });
 
 test("can play a single sound", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Valid sound"));
-  expect(
+  expect(testAudioContext.createBufferSource).toHaveBeenCalledOnce();
-    MockAudioContext.testContext.createBufferSource,
-  ).toHaveBeenCalledOnce();
 });
+
 test("will ignore sounds that are not registered", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Invalid sound"));
-  expect(
+  expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
-    MockAudioContext.testContext.createBufferSource,
-  ).not.toHaveBeenCalled();
 });
 
 test("will use the correct device", () => {
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   render(
     <MediaDevicesContext.Provider
       value={{
@@ -103,6 +115,7 @@ test("will use the correct device", () => {
           selectedGroupId: "",
           available: new Map(),
           select: () => {},
+          useAsEarpiece: false,
         },
         videoInput: deviceStub,
         startUsingDeviceNames: () => {},
@@ -112,21 +125,46 @@ test("will use the correct device", () => {
       <TestComponent />
     </MediaDevicesContext.Provider>,
   );
-  expect(
+  expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
-    MockAudioContext.testContext.createBufferSource,
+  expect(testAudioContext.setSinkId).toHaveBeenCalledWith("chosen-device");
-  ).not.toHaveBeenCalled();
-  expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
-    "chosen-device",
-  );
 });
 
 test("will use the correct volume level", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   soundEffectVolumeSetting.setValue(0.33);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Valid sound"));
-  expect(
+  expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
-    MockAudioContext.testContext.gain.gain.setValueAtTime,
+    0.33,
-  ).toHaveBeenCalledWith(0.33, 0);
+    0,
+  );
+  expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(0, 0);
+});
+
+test("will use the pan if earpice is selected", async () => {
+  const { findByText } = render(
+    <MediaDevicesContext.Provider
+      value={{
+        audioInput: deviceStub,
+        audioOutput: {
+          selectedId: "chosen-device",
+          selectedGroupId: "",
+          available: new Map(),
+          select: () => {},
+          useAsEarpiece: true,
+        },
+        videoInput: deviceStub,
+        startUsingDeviceNames: () => {},
+        stopUsingDeviceNames: () => {},
+      }}
+    >
+      <TestComponent />
+    </MediaDevicesContext.Provider>,
+  );
+  await user.click(await findByText("Valid sound"));
+  expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(1, 0);
+
+  expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
+    soundEffectVolumeSetting.getValue() * 0.1,
+    0,
+  );
 });

src/useAudioContext.tsx

@@ -12,7 +12,10 @@ import {
   soundEffectVolume as soundEffectVolumeSetting,
   useSetting,
 } from "./settings/settings";
-import { useMediaDevices } from "./livekit/MediaDevicesContext";
+import {
+  useEarpieceAudioConfig,
+  useMediaDevices,
+} from "./livekit/MediaDevicesContext";
 import { type PrefetchedSounds } from "./soundUtils";
 
 /**
@@ -28,12 +31,15 @@ async function playSound(
   ctx: AudioContext,
   buffer: AudioBuffer,
   volume: number,
+  stereoPan: number,
 ): Promise<void> {
   const gain = ctx.createGain();
   gain.gain.setValueAtTime(volume, 0);
+  const pan = ctx.createStereoPanner();
+  pan.pan.setValueAtTime(stereoPan, 0);
   const src = ctx.createBufferSource();
   src.buffer = buffer;
-  src.connect(gain).connect(ctx.destination);
+  src.connect(gain).connect(pan).connect(ctx.destination);
   const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
   src.start();
   return p;
@@ -63,8 +69,9 @@ interface UseAudioContext<S> {
 export function useAudioContext<S extends string>(
   props: Props<S>,
 ): UseAudioContext<S> | null {
-  const [effectSoundVolume] = useSetting(soundEffectVolumeSetting);
+  const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
-  const devices = useMediaDevices();
+  const { audioOutput } = useMediaDevices();
+
   const [audioContext, setAudioContext] = useState<AudioContext>();
   const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
 
@@ -106,23 +113,30 @@ export function useAudioContext<S extends string>(
     if (audioContext && "setSinkId" in audioContext) {
       // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
       // @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
-      audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
+      audioContext.setSinkId(audioOutput.selectedId).catch((ex) => {
         logger.warn("Unable to change sink for audio context", ex);
       });
     }
-  }, [audioContext, devices]);
+  }, [audioContext, audioOutput.selectedId]);
+  const { pan: earpiecePan, volume: earpieceVolume } = useEarpieceAudioConfig();
 
   // Don't return a function until we're ready.
   if (!audioContext || !audioBuffers || props.muted) {
     return null;
   }
+
   return {
     playSound: async (name): Promise<void> => {
       if (!audioBuffers[name]) {
         logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
         return;
       }
-      return playSound(audioContext, audioBuffers[name], effectSoundVolume);
+      return playSound(
+        audioContext,
+        audioBuffers[name],
+        soundEffectVolume * earpieceVolume,
+        earpiecePan,
+      );
     },
   };
 }

src/utils/test.ts

@@ -27,12 +27,14 @@ import {
   type RemoteParticipant,
   type RemoteTrackPublication,
   type Room as LivekitRoom,
+  Track,
 } from "livekit-client";
 import { randomUUID } from "crypto";
 import {
   type RoomAndToDeviceEvents,
   type RoomAndToDeviceEventsHandlerMap,
 } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
+import { type TrackReference } from "@livekit/components-core";
 
 import {
   LocalUserMediaViewModel,
@@ -309,3 +311,24 @@ export class MockRTCSession extends TypedEventEmitter<
     return this;
   }
 }
+
+export const mockTrack = (identity: string): TrackReference =>
+  ({
+    participant: {
+      identity,
+    },
+    publication: {
+      kind: Track.Kind.Audio,
+      source: "mic",
+      trackSid: "123",
+      track: {
+        attach: vi.fn(),
+        detach: vi.fn(),
+        setAudioContext: vi.fn(),
+        setWebAudioPlugins: vi.fn(),
+        setVolume: vi.fn(),
+      },
+    },
+    track: {},
+    source: {},
+  }) as unknown as TrackReference;