studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add custom audio renderer for iPhone earpiece and only render joined participants (#3249)

Browse Source 
* Add custom audio renderer to only render joined participants & add ios earpice workaround

fix left right to match chromium + safari
(firefox is swapped)

earpice as setting

Simpler code and documentation
The doc explains, what this class actually does and why it is so complicated.

Signed-off-by: Timo K <toger5@hotmail.de>

use only one audioContext, remove (non working) standby fallback

* Add tests

* use optional audio context and effect to initiate it + review
This commit is contained in:
Timo
2025-05-15 20:46:39 +02:00
committed by GitHub
parent 86d80630c1
commit b5fe55aef2
17 changed files with 588 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
locales/en/app.json
View File

@@ -61,6 +61,7 @@
"video": "Video"
},
"developer_mode": {
"always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms",
"crypto_version": "Crypto version: {{version}}",
"debug_tile_layout_label": "Debug tile layout",
"device_id": "Device ID: {{id}}",
@@ -174,6 +175,7 @@
"camera_numbered": "Camera {{n}}",
"default": "Default",
"default_named": "Default <2>({{name}})</2>",
"earpiece": "Earpiece",
"microphone": "Microphone",
"microphone_numbered": "Microphone {{n}}",
"speaker": "Speaker",

2
playwright/access.spec.ts
View File

@@ -49,7 +49,7 @@ test("Sign up a new account, then login, then logout", async ({ browser }) => {
// logout
await returningUserPage.getByTestId("usermenu_open").click();
await returningUserPage.locator('[data-test-id="usermenu_logout"]').click();
await returningUserPage.locator('[data-testid="usermenu_logout"]').click();
await expect(
returningUserPage.getByRole("link", { name: "Log In" }),

2
src/UserMenu.tsx
View File

@@ -119,7 +119,7 @@ export const UserMenu: FC<Props> = ({
key={key}
Icon={Icon}
label={label}
data-test-id={dataTestid}
data-testid={dataTestid}
onSelect={() => onAction(key)}
/>
))}

104
src/livekit/MatrixAudioRenderer.test.tsx Normal file
View File

@@ -0,0 +1,104 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { render } from "@testing-library/react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import {
getTrackReferenceId,
type TrackReference,
} from "@livekit/components-core";
import { type RemoteAudioTrack } from "livekit-client";
import { type ReactNode } from "react";
import { useTracks } from "@livekit/components-react";
import { testAudioContext } from "../useAudioContext.test";
import * as MediaDevicesContext from "./MediaDevicesContext";
import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
import { mockTrack } from "../utils/test";
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
beforeEach(() => {
vi.stubGlobal("AudioContext", TestAudioContextConstructor);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
vi.mock("@livekit/components-react", async (importOriginal) => {
return {
...(await importOriginal()),
AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
return (
<audio data-testid={"audio"}>
{getTrackReferenceId(props.trackRef)}
</audio>
);
},
useTracks: vi.fn(),
};
});
const tracks = [mockTrack("test:123")];
vi.mocked(useTracks).mockReturnValue(tracks);
it("should render for member", () => {
const { container, queryAllByTestId } = render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(1);
});
it("should not render without member", () => {
const { container, queryAllByTestId } = render(
<MatrixAudioRenderer
members={[{ sender: "othermember", deviceId: "123" }] as CallMembership[]}
/>,
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(0);
});
it("should not setup audioContext gain and pan if there is no need to.", () => {
render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]);
expect(testAudioContext.gain.gain.value).toEqual(1);
expect(testAudioContext.pan.pan.value).toEqual(0);
});
it("should setup audioContext gain and pan", () => {
vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
pan: 1,
volume: 0.1,
});
render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalled();
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled();
expect(testAudioContext.gain.gain.value).toEqual(0.1);
expect(testAudioContext.pan.pan.value).toEqual(1);
});

212
src/livekit/MatrixAudioRenderer.tsx Normal file
View File

@@ -0,0 +1,212 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { getTrackReferenceId } from "@livekit/components-core";
import { type RemoteAudioTrack, Track } from "livekit-client";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import {
useTracks,
AudioTrack,
type AudioTrackProps,
} from "@livekit/components-react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { useEarpieceAudioConfig } from "./MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
export interface MatrixAudioRendererProps {
/**
* The list of participants to render audio for.
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session.
*/
members: CallMembership[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
* If set to `true`, the server will stop sending audio track data to the client.
*/
muted?: boolean;
}
/**
* The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
* It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
* @example
* ```tsx
* <LiveKitRoom>
* <MatrixAudioRenderer />
* </LiveKitRoom>
* ```
* @public
*/
export function MatrixAudioRenderer({
members,
muted,
}: MatrixAudioRendererProps): ReactNode {
const validIdentities = useMemo(
() =>
new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
[members],
);
const loggedInvalidIdentities = useRef(new Set<string>());
/**
* Log an invalid livekit track identity.
* A invalid identity is one that does not match any of the matrix rtc members.
*
* @param identity The identity of the track that is invalid
* @param validIdentities The list of valid identities
*/
const logInvalid = (identity: string, validIdentities: Set<string>): void => {
if (loggedInvalidIdentities.current.has(identity)) return;
logger.warn(
`Audio track ${identity} has no matching matrix call member`,
`current members: ${Array.from(validIdentities.values())}`,
`track will not get rendered`,
);
loggedInvalidIdentities.current.add(identity);
};
const tracks = useTracks(
[
Track.Source.Microphone,
Track.Source.ScreenShareAudio,
Track.Source.Unknown,
],
{
updateOnlyOn: [],
onlySubscribed: true,
},
).filter((ref) => {
const isValid = validIdentities?.has(ref.participant.identity);
if (!isValid && !ref.participant.isLocal)
logInvalid(ref.participant.identity, validIdentities);
return (
!ref.participant.isLocal &&
ref.publication.kind === Track.Kind.Audio &&
isValid
);
});
// This component is also (in addition to the "only play audio for connected members" logic above)
// responsible for mimicking earpiece audio on iPhones.
// The Safari audio devices enumeration does not expose an earpiece audio device.
// We alternatively use the audioContext pan node to only use one of the stereo channels.
// This component does get additionally complicated because of a Safari bug.
// (see: https://bugs.webkit.org/show_bug.cgi?id=251532
// and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
// and https://bugs.webkit.org/show_bug.cgi?id=231105)
//
// AudioContext gets stopped if the webview gets moved into the background.
// Once the phone is in standby audio playback will stop.
// So we can only use the pan trick only works is the phone is not in standby.
// If earpiece mode is not used we do not use audioContext to allow standby playback.
// shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
const shouldUseAudioContext = stereoPan !== 0;
// initialize the potentially used audio context.
const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
undefined,
);
useEffect(() => {
const ctx = new AudioContext();
setAudioContext(ctx);
return (): void => {
void ctx.close();
};
}, []);
const audioNodes = useMemo(
() => ({
gain: audioContext?.createGain(),
pan: audioContext?.createStereoPanner(),
}),
[audioContext],
);
// Simple effects to update the gain and pan node based on the props
useEffect(() => {
if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
}, [audioNodes.pan, stereoPan]);
useEffect(() => {
if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
}, [audioNodes.gain, volumeFactor]);
return (
// We add all audio elements into one <div> for the browser developer tool experience/tidyness.
<div style={{ display: "none" }}>
{tracks.map((trackRef) => (
<AudioTrackWithAudioNodes
key={getTrackReferenceId(trackRef)}
trackRef={trackRef}
muted={muted}
audioContext={shouldUseAudioContext ? audioContext : undefined}
audioNodes={audioNodes}
/>
))}
</div>
);
}
interface StereoPanAudioTrackProps {
muted?: boolean;
audioContext?: AudioContext;
audioNodes: {
gain?: GainNode;
pan?: StereoPannerNode;
};
}
/**
* This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
* It main purpose is to remount the AudioTrack component when switching from
* audiooContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param param0
* @returns
*/
function AudioTrackWithAudioNodes({
trackRef,
muted,
audioContext,
audioNodes,
...props
}: StereoPanAudioTrackProps &
AudioTrackProps &
React.RefAttributes<HTMLAudioElement>): ReactNode {
// This is used to unmount/remount the AudioTrack component.
// Mounting needs to happen after the audioContext is set.
// (adding the audio context when already mounted did not work outside strict mode)
const [trackReady, setTrackReady] = useReactiveState(
() => false,
// We only want the track to reset once both (audioNodes and audioContext) are set.
// for unsetting the audioContext its enough if one of the the is undefined.
[audioContext && audioNodes],
);
useEffect(() => {
if (!trackRef || trackReady) return;
const track = trackRef.publication.track as RemoteAudioTrack;
const useContext = audioContext && audioNodes.gain && audioNodes.pan;
track.setAudioContext(useContext ? audioContext : undefined);
track.setWebAudioPlugins(
useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
);
setTrackReady(true);
}, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
return (
trackReady && <AudioTrack trackRef={trackRef} muted={muted} {...props} />
);
}

74
src/livekit/MediaDevicesContext.tsx
View File

@@ -26,12 +26,16 @@ import {
audioInput as audioInputSetting,
audioOutput as audioOutputSetting,
videoInput as videoInputSetting,
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
type Setting,
} from "../settings/settings";
export const EARPIECE_CONFIG_ID = "earpiece-id";
export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number }
| { type: "earpiece" }
| { type: "default"; name: string | null };
export interface MediaDevice {
@@ -40,6 +44,11 @@ export interface MediaDevice {
*/
available: Map<string, DeviceLabel>;
selectedId: string | undefined;
/**
* An additional device configuration that makes us use only one channel of the
* output device and a reduced volume.
*/
useAsEarpiece: boolean | undefined;
/**
* The group ID of the selected device.
*/
@@ -65,6 +74,7 @@ function useMediaDevice(
): MediaDevice {
// Make sure we don't needlessly reset to a device observer without names,
// once permissions are already given
const [alwaysShowIphoneEarpice] = useSetting(alwaysShowIphoneEarpieceSetting);
const hasRequestedPermissions = useRef(false);
const requestPermissions = usingNames || hasRequestedPermissions.current;
hasRequestedPermissions.current ||= usingNames;
@@ -102,27 +112,39 @@ function useMediaDevice(
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
// We also create this if we do not have any available devices, so that
// we can use the default or the earpiece.
const showEarpiece =
navigator.userAgent.match("iPhone") || alwaysShowIphoneEarpice;
if (
kind === "audiooutput" &&
available.size &&
!available.has("") &&
!available.has("default")
!available.has("default") &&
(available.size || showEarpiece)
)
available = new Map([
["", { type: "default", name: availableRaw[0]?.label || null }],
...available,
]);
if (kind === "audiooutput" && showEarpiece)
// On IPhones we have to create a virtual earpiece device, because
// the earpiece is not available as a device ID.
available = new Map([
...available,
[EARPIECE_CONFIG_ID, { type: "earpiece" }],
]);
// Note: creating virtual default input devices would be another problem
// entirely, because requesting a media stream from deviceId "" won't
// automatically track the default device.
return available;
}),
),
[kind, deviceObserver$],
[alwaysShowIphoneEarpice, deviceObserver$, kind],
),
);
const [preferredId, select] = useSetting(setting);
const [preferredId, setPreferredId] = useSetting(setting);
const [asEarpice, setAsEarpiece] = useState(false);
const selectedId = useMemo(() => {
if (available.size) {
// If the preferred device is available, use it. Or if every available
@@ -138,6 +160,7 @@ function useMediaDevice(
}
return undefined;
}, [available, preferredId]);
const selectedGroupId = useObservableEagerState(
useMemo(
() =>
@@ -151,14 +174,27 @@ function useMediaDevice(
),
);
const select = useCallback(
(id: string) => {
if (id === EARPIECE_CONFIG_ID) {
setAsEarpiece(true);
} else {
setAsEarpiece(false);
setPreferredId(id);
}
},
[setPreferredId],
);
return useMemo(
() => ({
available,
selectedId,
useAsEarpiece: asEarpice,
selectedGroupId,
select,
}),
[available, selectedId, selectedGroupId, select],
[available, selectedId, asEarpice, selectedGroupId, select],
);
}
@@ -167,6 +203,7 @@ export const deviceStub: MediaDevice = {
selectedId: undefined,
selectedGroupId: undefined,
select: () => {},
useAsEarpiece: false,
};
export const devicesStub: MediaDevices = {
audioInput: deviceStub,
@@ -255,3 +292,30 @@ export const useMediaDeviceNames = (
return context.stopUsingDeviceNames;
}
}, [context, enabled]);
/**
* A convenience hook to get the audio node configuration for the earpiece.
* It will check the `useAsEarpiece` of the `audioOutput` device and return
* the appropriate pan and volume values.
*
* @returns pan and volume values for the earpiece audio node configuration.
*/
export const useEarpieceAudioConfig = (): {
pan: number;
volume: number;
} => {
const { audioOutput } = useMediaDevices();
// We use only the right speaker (pan = 1) for the earpiece.
// This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone)
const pan = useMemo(
() => (audioOutput.useAsEarpiece ? 1 : 0),
[audioOutput.useAsEarpiece],
);
// We also do lower the volume by a factor of 10 to optimize for the usecase where
// a user is holding the phone to their ear.
const volume = useMemo(
() => (audioOutput.useAsEarpiece ? 0.1 : 1),
[audioOutput.useAsEarpiece],
);
return { pan, volume };
};

17
src/room/InCallView.test.tsx
View File

@@ -21,11 +21,7 @@ import { ConnectionState, type LocalParticipant } from "livekit-client";
import { of } from "rxjs";
import { BrowserRouter } from "react-router-dom";
import { TooltipProvider } from "@vector-im/compound-web";
import {
RoomAudioRenderer,
RoomContext,
useLocalParticipant,
} from "@livekit/components-react";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type MuteStates } from "./MuteStates";
@@ -48,6 +44,7 @@ import {
} from "../settings/settings";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
// vi.hoisted(() => {
// localStorage = {} as unknown as Storage;
@@ -65,6 +62,7 @@ vi.mock("../tile/GridTile");
vi.mock("../tile/SpotlightTile");
vi.mock("@livekit/components-react");
vi.mock("../e2ee/sharedKeyManagement");
vi.mock("../livekit/MatrixAudioRenderer");
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
}));
@@ -81,13 +79,15 @@ const roomMembers = new Map([carol].map((p) => [p.userId, p]));
const roomId = "!foo:bar";
let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
beforeEach(() => {
vi.clearAllMocks();
// RoomAudioRenderer is tested separately.
// MatrixAudioRenderer is tested separately.
(
RoomAudioRenderer as MockedFunction<typeof RoomAudioRenderer>
MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
).mockImplementation((_props) => {
return <div>mocked: RoomAudioRenderer</div>;
return <div>mocked: MatrixAudioRenderer</div>;
});
(
useLocalParticipant as MockedFunction<typeof useLocalParticipant>
@@ -98,7 +98,6 @@ beforeEach(() => {
localParticipant: localRtcMember as unknown as LocalParticipant,
}) as unknown as ReturnType<typeof useLocalParticipant>,
);
useRoomEncryptionSystemMock =
useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });

12
src/room/InCallView.tsx
View File

@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
RoomAudioRenderer,
RoomContext,
useLocalParticipant,
} from "@livekit/components-react";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { Text } from "@vector-im/compound-web";
import { ConnectionState, type Room } from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk";
@@ -107,6 +103,7 @@ import {
import { ReactionsReader } from "../reactions/ReactionsReader";
import { ConnectionLostError } from "../utils/errors.ts";
import { useTypedEventEmitter } from "../useEvents.ts";
import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -726,7 +723,10 @@ export const InCallView: FC<InCallViewProps> = ({
</Text>
)
}
<RoomAudioRenderer muted={muteAllAudio} />
<MatrixAudioRenderer
members={rtcSession.memberships}
muted={muteAllAudio}
/>
{renderContent()}
<CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
<ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />

1
src/room/MuteStates.test.tsx
View File

@@ -79,6 +79,7 @@ function mockDevices(available: Map<string, DeviceLabel>): MediaDevice {
selectedId: "",
selectedGroupId: "",
select: (): void => {},
useAsEarpiece: false,
};
}

2
src/room/__snapshots__/InCallView.test.tsx.snap
View File

@@ -9,7 +9,7 @@ exports[`InCallView > rendering > renders 1`] = `
class="header filler"
/>
<div>
mocked: RoomAudioRenderer
mocked: MatrixAudioRenderer
</div>
<div
class="scrollingGrid grid"

18
src/settings/DeveloperSettingsTab.tsx
View File

@@ -18,6 +18,7 @@ import {
useNewMembershipManager as useNewMembershipManagerSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
muteAllAudio as muteAllAudioSetting,
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
} from "./settings";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
@@ -46,6 +47,9 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
useNewMembershipManagerSetting,
);
const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
alwaysShowIphoneEarpieceSetting,
);
const [
useExperimentalToDeviceTransport,
setUseExperimentalToDeviceTransport,
@@ -192,6 +196,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
[setMuteAllAudio],
)}
/>
</FieldRow>{" "}
<FieldRow>
<InputField
id="alwaysShowIphoneEarpiece"
type="checkbox"
label={t("developer_mode.always_show_iphone_earpiece")}
checked={alwaysShowIphoneEarpiece}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setAlwaysShowIphoneEarpiece(event.target.checked);
},
[setAlwaysShowIphoneEarpiece],
)}
/>{" "}
</FieldRow>
{livekitRoom ? (
<>

30
src/settings/DeviceSelection.tsx
View File

@@ -22,17 +22,20 @@ import {
} from "@vector-im/compound-web";
import { Trans, useTranslation } from "react-i18next";
import { type MediaDevice } from "../livekit/MediaDevicesContext";
import {
EARPIECE_CONFIG_ID,
type MediaDevice,
} from "../livekit/MediaDevicesContext";
import styles from "./DeviceSelection.module.css";
interface Props {
devices: MediaDevice;
device: MediaDevice;
title: string;
numberedLabel: (number: number) => string;
}
export const DeviceSelection: FC<Props> = ({
devices,
device,
title,
numberedLabel,
}) => {
@@ -40,12 +43,13 @@ export const DeviceSelection: FC<Props> = ({
const groupId = useId();
const onChange = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
devices.select(e.target.value);
device.select(e.target.value);
},
[devices],
[device],
);
if (devices.available.size == 0) return null;
// There is no need to show the menu if there is no choice that can be made.
if (device.available.size <= 1) return null;
return (
<div className={styles.selection}>
@@ -60,7 +64,7 @@ export const DeviceSelection: FC<Props> = ({
</Heading>
<Separator className={styles.separator} />
<div className={styles.options}>
{[...devices.available].map(([id, label]) => {
{[...device.available].map(([id, label]) => {
let labelText: ReactNode;
switch (label.type) {
case "name":
@@ -85,6 +89,16 @@ export const DeviceSelection: FC<Props> = ({
</Trans>
);
break;
case "earpiece":
labelText = t("settings.devices.earpiece");
break;
}
let isSelected = false;
if (device.useAsEarpiece) {
isSelected = id === EARPIECE_CONFIG_ID;
} else {
isSelected = id === device.selectedId;
}
return (
@@ -93,7 +107,7 @@ export const DeviceSelection: FC<Props> = ({
name={groupId}
control={
<RadioControl
checked={id === devices.selectedId}
checked={isSelected}
onChange={onChange}
value={id}
/>

8
src/settings/SettingsModal.tsx
View File

@@ -98,7 +98,6 @@ export const SettingsModal: FC<Props> = ({
useMediaDeviceNames(devices, open);
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
const [showDeveloperSettingsTab] = useSetting(developerMode);
const { available: isRageshakeAvailable } = useSubmitRageshake();
@@ -110,17 +109,18 @@ export const SettingsModal: FC<Props> = ({
<>
<Form>
<DeviceSelection
devices={devices.audioInput}
device={devices.audioInput}
title={t("settings.devices.microphone")}
numberedLabel={(n) =>
t("settings.devices.microphone_numbered", { n })
}
/>
<DeviceSelection
devices={devices.audioOutput}
device={devices.audioOutput}
title={t("settings.devices.speaker")}
numberedLabel={(n) => t("settings.devices.speaker_numbered", { n })}
/>
<div className={styles.volumeSlider}>
<label>{t("settings.audio_tab.effect_volume_label")}</label>
<p>{t("settings.audio_tab.effect_volume_description")}</p>
@@ -146,7 +146,7 @@ export const SettingsModal: FC<Props> = ({
<>
<Form>
<DeviceSelection
devices={devices.videoInput}
device={devices.videoInput}
title={t("settings.devices.camera")}
numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
/>

8
src/settings/settings.ts
View File

@@ -44,6 +44,9 @@ export class Setting<T> {
this._value$.next(value);
localStorage.setItem(this.key, JSON.stringify(value));
};
public readonly getValue = (): T => {
return this._value$.getValue();
};
}
/**
@@ -128,3 +131,8 @@ export const useExperimentalToDeviceTransport = new Setting<boolean>(
export const muteAllAudio = new Setting<boolean>("mute-all-audio", false);
export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);
export const alwaysShowIphoneEarpiece = new Setting<boolean>(
"always-show-iphone-earpice",
false,
);

136
src/useAudioContext.test.tsx
View File

@@ -5,10 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test, vitest, afterEach } from "vitest";
import { expect, vi, afterEach, beforeEach, test } from "vitest";
import { type FC } from "react";
import { render } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import userEvent, { type UserEvent } from "@testing-library/user-event";
import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
import { useAudioContext } from "./useAudioContext";
@@ -39,61 +39,73 @@ const TestComponent: FC = () => {
);
};
class MockAudioContext {
public static testContext: MockAudioContext;
public constructor() {
MockAudioContext.testContext = this;
}
public gain = vitest.mocked(
{
connect: () => {},
gain: {
setValueAtTime: vitest.fn(),
},
const gainNode = vi.mocked(
{
connect: (node: AudioNode) => node,
gain: {
setValueAtTime: vi.fn(),
value: 1,
},
true,
);
public setSinkId = vitest.fn().mockResolvedValue(undefined);
public decodeAudioData = vitest.fn().mockReturnValue(1);
public createBufferSource = vitest.fn().mockReturnValue(
vitest.mocked({
},
true,
);
const panNode = vi.mocked(
{
connect: (node: AudioNode) => node,
pan: {
setValueAtTime: vi.fn(),
value: 0,
},
},
true,
);
/**
* A shared audio context test instance.
* It can also be used to mock the `AudioContext` constructor in tests:
* `vi.stubGlobal("AudioContext", () => testAudioContext);`
*/
export const testAudioContext = {
gain: gainNode,
pan: panNode,
setSinkId: vi.fn().mockResolvedValue(undefined),
decodeAudioData: vi.fn().mockReturnValue(1),
createBufferSource: vi.fn().mockReturnValue(
vi.mocked({
connect: (v: unknown) => v,
start: () => {},
addEventListener: (_name: string, cb: () => void) => cb(),
}),
);
public createGain = vitest.fn().mockReturnValue(this.gain);
public close = vitest.fn().mockResolvedValue(undefined);
}
),
createGain: vi.fn().mockReturnValue(gainNode),
createStereoPanner: vi.fn().mockReturnValue(panNode),
close: vi.fn().mockResolvedValue(undefined),
};
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
let user: UserEvent;
beforeEach(() => {
vi.stubGlobal("AudioContext", TestAudioContextConstructor);
user = userEvent.setup();
});
afterEach(() => {
vitest.unstubAllGlobals();
vi.unstubAllGlobals();
vi.clearAllMocks();
});
test("can play a single sound", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).toHaveBeenCalledOnce();
expect(testAudioContext.createBufferSource).toHaveBeenCalledOnce();
});
test("will ignore sounds that are not registered", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Invalid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
});
test("will use the correct device", () => {
vitest.stubGlobal("AudioContext", MockAudioContext);
render(
<MediaDevicesContext.Provider
value={{
@@ -103,6 +115,7 @@ test("will use the correct device", () => {
selectedGroupId: "",
available: new Map(),
select: () => {},
useAsEarpiece: false,
},
videoInput: deviceStub,
startUsingDeviceNames: () => {},
@@ -112,21 +125,46 @@ test("will use the correct device", () => {
<TestComponent />
</MediaDevicesContext.Provider>,
);
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
"chosen-device",
);
expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
expect(testAudioContext.setSinkId).toHaveBeenCalledWith("chosen-device");
});
test("will use the correct volume level", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
soundEffectVolumeSetting.setValue(0.33);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.gain.gain.setValueAtTime,
).toHaveBeenCalledWith(0.33, 0);
expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
0.33,
0,
);
expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(0, 0);
});
test("will use the pan if earpice is selected", async () => {
const { findByText } = render(
<MediaDevicesContext.Provider
value={{
audioInput: deviceStub,
audioOutput: {
selectedId: "chosen-device",
selectedGroupId: "",
available: new Map(),
select: () => {},
useAsEarpiece: true,
},
videoInput: deviceStub,
startUsingDeviceNames: () => {},
stopUsingDeviceNames: () => {},
}}
>
<TestComponent />
</MediaDevicesContext.Provider>,
);
await user.click(await findByText("Valid sound"));
expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(1, 0);
expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
soundEffectVolumeSetting.getValue() * 0.1,
0,
);
});

28
src/useAudioContext.tsx
View File

@@ -12,7 +12,10 @@ import {
soundEffectVolume as soundEffectVolumeSetting,
useSetting,
} from "./settings/settings";
import { useMediaDevices } from "./livekit/MediaDevicesContext";
import {
useEarpieceAudioConfig,
useMediaDevices,
} from "./livekit/MediaDevicesContext";
import { type PrefetchedSounds } from "./soundUtils";
/**
@@ -28,12 +31,15 @@ async function playSound(
ctx: AudioContext,
buffer: AudioBuffer,
volume: number,
stereoPan: number,
): Promise<void> {
const gain = ctx.createGain();
gain.gain.setValueAtTime(volume, 0);
const pan = ctx.createStereoPanner();
pan.pan.setValueAtTime(stereoPan, 0);
const src = ctx.createBufferSource();
src.buffer = buffer;
src.connect(gain).connect(ctx.destination);
src.connect(gain).connect(pan).connect(ctx.destination);
const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
src.start();
return p;
@@ -63,8 +69,9 @@ interface UseAudioContext<S> {
export function useAudioContext<S extends string>(
props: Props<S>,
): UseAudioContext<S> | null {
const [effectSoundVolume] = useSetting(soundEffectVolumeSetting);
const devices = useMediaDevices();
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
const { audioOutput } = useMediaDevices();
const [audioContext, setAudioContext] = useState<AudioContext>();
const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
@@ -106,23 +113,30 @@ export function useAudioContext<S extends string>(
if (audioContext && "setSinkId" in audioContext) {
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
audioContext.setSinkId(audioOutput.selectedId).catch((ex) => {
logger.warn("Unable to change sink for audio context", ex);
});
}
}, [audioContext, devices]);
}, [audioContext, audioOutput.selectedId]);
const { pan: earpiecePan, volume: earpieceVolume } = useEarpieceAudioConfig();
// Don't return a function until we're ready.
if (!audioContext || !audioBuffers || props.muted) {
return null;
}
return {
playSound: async (name): Promise<void> => {
if (!audioBuffers[name]) {
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
return;
}
return playSound(audioContext, audioBuffers[name], effectSoundVolume);
return playSound(
audioContext,
audioBuffers[name],
soundEffectVolume * earpieceVolume,
earpiecePan,
);
},
};
}

23
src/utils/test.ts
View File

@@ -27,12 +27,14 @@ import {
type RemoteParticipant,
type RemoteTrackPublication,
type Room as LivekitRoom,
Track,
} from "livekit-client";
import { randomUUID } from "crypto";
import {
type RoomAndToDeviceEvents,
type RoomAndToDeviceEventsHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type TrackReference } from "@livekit/components-core";
import {
LocalUserMediaViewModel,
@@ -309,3 +311,24 @@ export class MockRTCSession extends TypedEventEmitter<
return this;
}
}
export const mockTrack = (identity: string): TrackReference =>
({
participant: {
identity,
},
publication: {
kind: Track.Kind.Audio,
source: "mic",
trackSid: "123",
track: {
attach: vi.fn(),
detach: vi.fn(),
setAudioContext: vi.fn(),
setWebAudioPlugins: vi.fn(),
setVolume: vi.fn(),
},
},
track: {},
source: {},
}) as unknown as TrackReference;