diff --git a/src/controls.ts b/src/controls.ts index 6a050cb0..86de0ace 100644 --- a/src/controls.ts +++ b/src/controls.ts @@ -33,12 +33,36 @@ export interface Controls { showNativeOutputDevicePicker?: () => void; } +/** + * Output Audio device when using the controlled audio output mode (mobile). + */ export interface OutputDevice { id: string; name: string; + /** + * Reverse engineered: + * - on iOS always true if output is routed to speaker. In other case iOS on declare a `dummy` id device. + * In that case then ElementCalls manually append a earpiece device with id `EARPIECE_CONFIG_ID` anb `{ type: "earpiece" }` + * - on Android this is unused. + */ forEarpiece?: boolean; + /** + * Reverse engineered: + * - on iOS always undefined + * - on Android true for the `TYPE_BUILTIN_EARPIECE` + */ isEarpiece?: boolean; + /** + * Reverse engineered: + * - on iOS always true if output is routed to speaker. In other case iOS on declare a `dummy` id device. + * - on Android true for the `TYPE_BUILTIN_SPEAKER` + */ isSpeaker?: boolean; + /** + * Reverse engineered: + * - on iOS always undefined. + * - on Android true for the `TYPE_BLUETOOTH_SCO` + */ isExternalHeadset?: boolean; } @@ -47,8 +71,16 @@ export interface OutputDevice { */ export const setPipEnabled$ = new Subject(); +/** + * Stores the list of available controlled audio output devices. + * This is set when the native code calls `setAvailableAudioDevices` with the list of available audio output devices. + */ export const availableOutputDevices$ = new Subject(); +/** + * Stores the current audio output device id. + * This is set when the native code calls `setAudioDevice` + */ export const outputDevice$ = new Subject(); /** @@ -80,16 +112,41 @@ window.controls = { setPipEnabled$.next(false); }, + /** + * Reverse engineered: + * + * - on iOS: + * This always a list of one thing. If current route output is speaker it returns + * the single `{"id":"Speaker","name":"Speaker","forEarpiece":true,"isSpeaker":true}` Notice that EC will + * also manually add a virtual earpiece device with id `EARPIECE_CONFIG_ID` and `{ type: "earpiece" }`. + * If the route output is not speaker then it will be `{id: 'dummy', name: 'dummy'}` + * + * + * - on Android: + * This is a list of all available output audio devices. The `id` is the Android AudioDeviceInfo.getId() + * and the `name` is based the Android AudioDeviceInfo.productName (mapped to static strings for known types) + * The `isEarpiece`, `isSpeaker` and `isExternalHeadset` are set based on the Android AudioDeviceInfo.type + * matching the corresponding types for earpiece, speaker and bluetooth headset. + */ setAvailableAudioDevices(devices: OutputDevice[]): void { - logger.info("setAvailableAudioDevices called from native:", devices); + logger.info( + "[MediaDevices controls] setAvailableAudioDevices called from native:", + devices, + ); availableOutputDevices$.next(devices); }, setAudioDevice(id: string): void { - logger.info("setAudioDevice called from native", id); + logger.info( + "[MediaDevices controls] setAudioDevice called from native", + id, + ); outputDevice$.next(id); }, setAudioEnabled(enabled: boolean): void { - logger.info("setAudioEnabled called from native:", enabled); + logger.info( + "[MediaDevices controls] setAudioEnabled called from native:", + enabled, + ); if (!setAudioEnabled$.observed) throw new Error( "Output controls are disabled. No setAudioEnabled$ observer", diff --git a/src/main.tsx b/src/main.tsx index 946e0238..6cbf75fa 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -67,6 +67,6 @@ Initializer.initBeforeReact() ); }) .catch((e) => { - logger.error("Failed to initialize app", e); + logger.error(`Failed to initialize app ${e.message}`, e); root.render(e.message); }); diff --git a/src/state/AndroidControlledAudioOutput.test.ts b/src/state/AndroidControlledAudioOutput.test.ts new file mode 100644 index 00000000..12b74052 --- /dev/null +++ b/src/state/AndroidControlledAudioOutput.test.ts @@ -0,0 +1,563 @@ +/* +Copyright 2026 Element Corp. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { it, vi, expect, beforeEach, afterEach, describe } from "vitest"; +import { firstValueFrom, of, Subject, take, toArray } from "rxjs"; +import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc"; + +import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts"; +import type { Controls, OutputDevice } from "../controls"; +import { ObservableScope } from "./ObservableScope"; +import { withTestScheduler } from "../utils/test"; + +// All the following device types are real device types that have been observed in the wild on Android devices, +// gathered from logs. +// There are no BT Speakers because they are currently filtered out by EXA (native layer) + +// A device type describing the speaker system (i.e. a mono speaker or stereo speakers) built in a device. +const SPEAKER_DEVICE: OutputDevice = { + id: "3", + name: "Built-in speaker", + isEarpiece: false, + isSpeaker: true, + isExternalHeadset: false, +}; + +// A device type describing the attached earphone speaker. +const EARPIECE_DEVICE: OutputDevice = { + id: "2", + name: "Built-in earpiece", + isEarpiece: true, + isSpeaker: false, + isExternalHeadset: false, +}; + +// A device type describing a Bluetooth device typically used for telephony +const BT_HEADSET_DEVICE: OutputDevice = { + id: "2226", + name: "Bluetooth - OpenMove by Shokz", + isEarpiece: false, + isSpeaker: false, + isExternalHeadset: true, +}; + +// A device type describing a USB audio headset. +const USB_HEADSET_DEVICE: OutputDevice = { + id: "29440", + name: "USB headset - USB-Audio - AB13X USB Audio", + isEarpiece: false, + isSpeaker: false, + isExternalHeadset: false, +}; + +// A device type describing a headset, which is the combination of a headphones and microphone +const WIRED_HEADSET_DEVICE: OutputDevice = { + id: "54509", + name: "Wired headset - 23117RA68G", + isEarpiece: false, + isSpeaker: false, + isExternalHeadset: false, +}; + +// A device type describing a pair of wired headphones +const WIRED_HEADPHONE_DEVICE: OutputDevice = { + id: "679", + name: "Wired headphones - TB02", + isEarpiece: false, + isSpeaker: false, + isExternalHeadset: false, +}; + +/** + * The base device list that is always present on Android devices. + * This list is ordered by the OS, the speaker is listed before the earpiece. + */ +const BASE_DEVICE_LIST = [SPEAKER_DEVICE, EARPIECE_DEVICE]; + +const BT_HEADSET_BASE_DEVICE_LIST = [BT_HEADSET_DEVICE, ...BASE_DEVICE_LIST]; + +const WIRED_HEADSET_BASE_DEVICE_LIST = [ + WIRED_HEADSET_DEVICE, + ...BASE_DEVICE_LIST, +]; + +/** + * A full device list containing all the observed device types in the wild on Android devices. + * Ordered as they would be ordered by the OS. + */ +const FULL_DEVICE_LIST = [ + BT_HEADSET_DEVICE, + USB_HEADSET_DEVICE, + WIRED_HEADSET_DEVICE, + WIRED_HEADPHONE_DEVICE, + ...BASE_DEVICE_LIST, +]; + +let testScope: ObservableScope; +let mockControls: Controls; + +beforeEach(() => { + testScope = new ObservableScope(); + mockControls = { + onAudioDeviceSelect: vi.fn(), + onOutputDeviceSelect: vi.fn(), + } as unknown as Controls; +}); + +afterEach(() => { + testScope.end(); +}); + +describe("Default selection", () => { + it("Default to speaker for video calls", async () => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(BASE_DEVICE_LIST), + testScope, + "video", + mockControls, + ); + + const emissions = await firstValueFrom( + controlledAudioOutput.selected$.pipe(take(1), toArray()), + ); + + expect(emissions).toEqual([ + { id: SPEAKER_DEVICE.id, virtualEarpiece: false }, + ]); + + [ + mockControls.onAudioDeviceSelect, + mockControls.onOutputDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id); + }); + }); + + it("Default to earpiece for audio calls for base config", async () => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(BASE_DEVICE_LIST), + testScope, + "audio", + mockControls, + ); + + const emissions = await firstValueFrom( + controlledAudioOutput.selected$.pipe(take(1), toArray()), + ); + + expect(emissions).toEqual([ + { id: EARPIECE_DEVICE.id, virtualEarpiece: false }, + ]); + + [ + mockControls.onAudioDeviceSelect, + mockControls.onOutputDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(EARPIECE_DEVICE.id); + }); + }); + + ["audio", "video"].forEach((callIntent) => { + it(`Default to BT headset for ${callIntent} calls if present`, async () => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(BT_HEADSET_BASE_DEVICE_LIST), + testScope, + callIntent, + mockControls, + ); + + const emissions = await firstValueFrom( + controlledAudioOutput.selected$.pipe(take(1), toArray()), + ); + + expect(emissions).toEqual([ + { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false }, + ]); + + [ + mockControls.onAudioDeviceSelect, + mockControls.onOutputDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id); + }); + }); + }); + + ["audio", "video"].forEach((callIntent) => { + it(`Default to wired headset for ${callIntent} calls if present`, async () => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(WIRED_HEADSET_BASE_DEVICE_LIST), + testScope, + callIntent, + mockControls, + ); + + const emissions = await firstValueFrom( + controlledAudioOutput.selected$.pipe(take(1), toArray()), + ); + + expect(emissions).toEqual([ + { id: WIRED_HEADSET_DEVICE.id, virtualEarpiece: false }, + ]); + + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledExactlyOnceWith( + WIRED_HEADSET_DEVICE.id, + ); + expect(mockControls.onOutputDeviceSelect).toHaveBeenCalledExactlyOnceWith( + WIRED_HEADSET_DEVICE.id, + ); + }); + }); +}); + +describe("Test mappings", () => { + it("Should map output device to correct AudioDeviceLabel", async () => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(FULL_DEVICE_LIST), + testScope, + undefined, + mockControls, + ); + + const availableDevices = await firstValueFrom( + controlledAudioOutput.available$.pipe(take(1)), + ); + + expect(availableDevices).toEqual( + new Map([ + [BT_HEADSET_DEVICE.id, { type: "name", name: BT_HEADSET_DEVICE.name }], + [ + USB_HEADSET_DEVICE.id, + { type: "name", name: USB_HEADSET_DEVICE.name }, + ], + [ + WIRED_HEADSET_DEVICE.id, + { type: "name", name: WIRED_HEADSET_DEVICE.name }, + ], + [ + WIRED_HEADPHONE_DEVICE.id, + { type: "name", name: WIRED_HEADPHONE_DEVICE.name }, + ], + [SPEAKER_DEVICE.id, { type: "speaker" }], + [EARPIECE_DEVICE.id, { type: "earpiece" }], + ]), + ); + }); +}); + +describe("Test select a device", () => { + it(`Switch to correct device `, () => { + withTestScheduler(({ cold, schedule, expectObservable, flush }) => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + cold("a", { a: FULL_DEVICE_LIST }), + testScope, + undefined, + mockControls, + ); + + schedule("-abc", { + a: () => controlledAudioOutput.select(EARPIECE_DEVICE.id), + b: () => controlledAudioOutput.select(USB_HEADSET_DEVICE.id), + c: () => controlledAudioOutput.select(SPEAKER_DEVICE.id), + }); + + expectObservable(controlledAudioOutput.selected$).toBe("abcd", { + // virtualEarpiece is always false on android. + // Initially the BT_HEADSET is selected. + a: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false }, + b: { id: EARPIECE_DEVICE.id, virtualEarpiece: false }, + c: { id: USB_HEADSET_DEVICE.id, virtualEarpiece: false }, + d: { id: SPEAKER_DEVICE.id, virtualEarpiece: false }, + }); + + flush(); + + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(4); + expect(mockFn).toHaveBeenNthCalledWith(1, BT_HEADSET_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(2, EARPIECE_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(3, USB_HEADSET_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(4, SPEAKER_DEVICE.id); + }); + }); + }); + + it(`manually switch then a bt headset is added`, () => { + withTestScheduler(({ cold, schedule, expectObservable, flush }) => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + cold("a--b", { + a: BASE_DEVICE_LIST, + b: BT_HEADSET_BASE_DEVICE_LIST, + }), + testScope, + "audio", + mockControls, + ); + + // Default was earpiece (audio call), let's switch to speaker + schedule("-a--", { + a: () => controlledAudioOutput.select(SPEAKER_DEVICE.id), + }); + + expectObservable(controlledAudioOutput.selected$).toBe("ab-c", { + // virtualEarpiece is always false on android. + // Initially the BT_HEADSET is selected. + a: { id: EARPIECE_DEVICE.id, virtualEarpiece: false }, + b: { id: SPEAKER_DEVICE.id, virtualEarpiece: false }, + c: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false }, + }); + + flush(); + + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(3); + expect(mockFn).toHaveBeenNthCalledWith(1, EARPIECE_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(2, SPEAKER_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(3, BT_HEADSET_DEVICE.id); + }); + }); + }); + + it(`Go back to the previously selected after the auto-switch device goes away`, () => { + withTestScheduler(({ cold, schedule, expectObservable, flush }) => { + const controlledAudioOutput = new AndroidControlledAudioOutput( + cold("a--b-c", { + a: BASE_DEVICE_LIST, + b: BT_HEADSET_BASE_DEVICE_LIST, + c: BASE_DEVICE_LIST, + }), + testScope, + "audio", + mockControls, + ); + + // Default was earpiece (audio call), let's switch to speaker + schedule("-a---", { + a: () => controlledAudioOutput.select(SPEAKER_DEVICE.id), + }); + + expectObservable(controlledAudioOutput.selected$).toBe("ab-c-d", { + // virtualEarpiece is always false on android. + // Initially the BT_HEADSET is selected. + a: { id: EARPIECE_DEVICE.id, virtualEarpiece: false }, + b: { id: SPEAKER_DEVICE.id, virtualEarpiece: false }, + c: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false }, + d: { id: SPEAKER_DEVICE.id, virtualEarpiece: false }, + }); + + flush(); + + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(4); + expect(mockFn).toHaveBeenNthCalledWith(1, EARPIECE_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(2, SPEAKER_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(3, BT_HEADSET_DEVICE.id); + expect(mockFn).toHaveBeenNthCalledWith(4, SPEAKER_DEVICE.id); + }); + }); + }); +}); + +describe("Available device changes", () => { + let availableSource$: Subject; + + const createAudioControlledOutput = ( + intent: RTCCallIntent, + ): AndroidControlledAudioOutput => { + return new AndroidControlledAudioOutput( + availableSource$, + testScope, + intent, + mockControls, + ); + }; + + beforeEach(() => { + availableSource$ = new Subject(); + }); + + it("When a BT headset is added, control should switch to use it", () => { + createAudioControlledOutput("video"); + + // Emit the base device list, the speaker should be selected + availableSource$.next(BASE_DEVICE_LIST); + // Initially speaker would be selected + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id); + }); + + // Emit a new device list with a BT device, the control should switch to it + availableSource$.next([BT_HEADSET_DEVICE, ...BASE_DEVICE_LIST]); + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockFn).toHaveBeenLastCalledWith(BT_HEADSET_DEVICE.id); + }); + }); + + // Android does not set `isExternalHeadset` to true for wired headphones, so we can't test this case.' + it.skip("When a wired headset is added, control should switch to use it", async () => { + const controlledAudioOutput = createAudioControlledOutput("video"); + + // Emit the base device list, the speaker should be selected + availableSource$.next(BASE_DEVICE_LIST); + + await firstValueFrom(controlledAudioOutput.selected$.pipe(take(1))); + // Initially speaker would be selected + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id); + }); + + // Emit a new device list with a wired headset, the control should switch to it + availableSource$.next([WIRED_HEADPHONE_DEVICE, ...BASE_DEVICE_LIST]); + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockFn).toHaveBeenLastCalledWith(WIRED_HEADPHONE_DEVICE.id); + }); + }); + + it("When the active bt headset is removed on audio call, control should switch to earpiece", () => { + createAudioControlledOutput("audio"); + + // Emit the BT headset device list, the BT headset should be selected + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + // Initially speaker would be selected + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id); + }); + + // Emit a new device list without the BT headset, the control should switch to the earpiece for + // audio calls + availableSource$.next(BASE_DEVICE_LIST); + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockFn).toHaveBeenLastCalledWith(EARPIECE_DEVICE.id); + }); + }); + + it("When the active bt headset is removed on video call, control should switch to speaker", () => { + createAudioControlledOutput("video"); + + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + + // Initially bt headset would be selected + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id); + }); + + // Emit a new device list without the BT headset, the control should switch to speaker for video call + availableSource$.next(BASE_DEVICE_LIST); + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockFn).toHaveBeenLastCalledWith(SPEAKER_DEVICE.id); + }); + }); + + it("Do not repeatidly set the same device", () => { + createAudioControlledOutput("video"); + + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + + // Initially bt headset would be selected + [ + mockControls.onOutputDeviceSelect, + mockControls.onAudioDeviceSelect, + ].forEach((mockFn) => { + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id); + }); + }); +}); + +describe("Scope management", () => { + it("Should stop emitting when scope ends", () => { + const aScope = new ObservableScope(); + const controlledAudioOutput = new AndroidControlledAudioOutput( + of(BASE_DEVICE_LIST), + aScope, + undefined, + mockControls, + ); + + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce(); + + aScope.end(); + + controlledAudioOutput.select(EARPIECE_DEVICE.id); + + expect(mockControls.onAudioDeviceSelect).not.toHaveBeenCalledTimes(2); + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce(); + }); + + it("Should stop updating when scope ends", () => { + const aScope = new ObservableScope(); + const availableSource$ = new Subject(); + new AndroidControlledAudioOutput( + availableSource$, + aScope, + undefined, + mockControls, + ); + + availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST); + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce(); + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledWith( + BT_HEADSET_DEVICE.id, + ); + + aScope.end(); + + availableSource$.next(BASE_DEVICE_LIST); + + expect(mockControls.onAudioDeviceSelect).not.toHaveBeenCalledTimes(2); + // Should have been called only once with the initial BT_HEADSET_DEVICE.id + expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/state/AndroidControlledAudioOutput.ts b/src/state/AndroidControlledAudioOutput.ts new file mode 100644 index 00000000..ce4974ff --- /dev/null +++ b/src/state/AndroidControlledAudioOutput.ts @@ -0,0 +1,358 @@ +/* +Copyright 2026 Element Corp. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { + distinctUntilChanged, + map, + merge, + type Observable, + scan, + startWith, + Subject, + tap, +} from "rxjs"; + +import { + type AudioOutputDeviceLabel, + type MediaDevice, + type SelectedAudioOutputDevice, +} from "./MediaDevices.ts"; +import type { ObservableScope } from "./ObservableScope.ts"; +import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc"; +import { type Controls, type OutputDevice } from "../controls.ts"; +import { type Behavior } from "./Behavior.ts"; + +type ControllerState = { + /** + * The list of available output devices, ordered by preference order (most preferred first). + */ + devices: OutputDevice[]; + /** + * Explicit user preference for the selected device. + */ + preferredDeviceId: string | undefined; + /** + * The effective selected device, always valid against available devices. + */ + selectedDeviceId: string | undefined; +}; + +/** + * The possible actions that can be performed on the controller, + * either by the user or by the system. + */ +type ControllerAction = + | { type: "selectDevice"; deviceId: string | undefined } + | { type: "deviceUpdated"; devices: OutputDevice[] }; +/** + * The implementation of the audio output media device for Android when using the controlled audio output mode. + * + * In this mode, the hosting application (e.g. Element Mobile) is responsible for providing the list of available audio output devices. + * There are some android specific logic compared to others: + * - AndroidControlledAudioOutput is the only one responsible for selecting the best output device. + * - On android, we don't listen to the selected device from native code (control.setAudioDevice). + * - If a new device is added or removed, this controller will determine the new selected device based + * on the available devices (that is ordered by preference order) and the user's selection (if any). + * + * Given the differences in how the native code is handling the audio routing on Android compared to iOS, + * we have this separate implementation. It allows us to have proper testing and avoid side effects + * from platform specific logic breaking the other platform's implementation. + */ +export class AndroidControlledAudioOutput implements MediaDevice< + AudioOutputDeviceLabel, + SelectedAudioOutputDevice +> { + private logger = rootLogger.getChild( + "[MediaDevices AndroidControlledAudioOutput]", + ); + + // STATE stream: the current state of the controller, including the list of available devices and the selected device. + private readonly controllerState$: Behavior; + + /** + * @inheritdoc + */ + public readonly available$: Behavior>; + + /** + * Effective selected device, always valid against available devices. + * + * On android, we don't listen to the selected device from native code (control.setAudioDevice). + * Instead, we determine the selected device ourselves based on the available devices and the user's selection (if any). + */ + public readonly selected$: Behavior; + + // COMMAND stream: user asks to select a device + private readonly selectDeviceCommand$ = new Subject(); + + public select(id: string): void { + this.logger.info(`select device: ${id}`); + this.selectDeviceCommand$.next(id); + } + + /** + * Creates an instance of AndroidControlledAudioOutput. + * + * @constructor + * @param controlledDevices$ - The list of available output devices coming from the hosting application, ordered by preference order (most preferred first). + * @param scope - The ObservableScope to create the Behaviors in. + * @param initialIntent - The initial call intent (e.g. "audio" or "video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls). + * @param controls - The controls provided by the hosting application to control the audio routing and notify of user actions. + */ + public constructor( + private readonly controlledDevices$: Observable, + private readonly scope: ObservableScope, + private initialIntent: RTCCallIntent | undefined = undefined, + controls: Controls, + ) { + this.controllerState$ = this.startObservingState$(); + + this.selected$ = this.effectiveSelectionFromState$(this.controllerState$); + + this.available$ = scope.behavior( + this.controllerState$.pipe( + map((state) => { + this.logger.info("available devices updated:", state.devices); + + return new Map( + state.devices.map((outputDevice) => { + return [outputDevice.id, mapDeviceToLabel(outputDevice)]; + }), + ); + }), + ), + ); + + // Effect 1: notify host when effective selection changes + this.selected$ + // It is a behavior so it has built-in distinct until change + .pipe(scope.bind()) + .subscribe((device) => { + // Let the hosting application know which output device has been selected. + if (device !== undefined) { + this.logger.info("onAudioDeviceSelect called:", device); + controls.onAudioDeviceSelect?.(device.id); + // Also invoke the deprecated callback for backward compatibility + // TODO: it appears that on Android the hosting application is only using the deprecated callback (onOutputDeviceSelect) + // and not the new one (onAudioDeviceSelect), we should clean this up and only have one callback for audio device selection. + controls.onOutputDeviceSelect?.(device.id); + } + }); + } + + private startObservingState$(): Behavior { + const initialState: ControllerState = { + devices: [], + preferredDeviceId: undefined, + selectedDeviceId: undefined, + }; + + // Merge the two possible inputs observable as a single + // stream of actions that will update the state of the controller. + const actions$: Observable = merge( + this.controlledDevices$.pipe( + map( + (devices) => + ({ type: "deviceUpdated", devices }) satisfies ControllerAction, + ), + ), + this.selectDeviceCommand$.pipe( + map( + (deviceId) => + ({ type: "selectDevice", deviceId }) satisfies ControllerAction, + ), + ), + ); + + const initialAction: ControllerAction = { + type: "deviceUpdated", + devices: [], + }; + + return this.scope.behavior( + actions$.pipe( + startWith(initialAction), + scan((state, action): ControllerState => { + switch (action.type) { + case "deviceUpdated": { + const chosenDevice = this.chooseEffectiveSelection({ + previousDevices: state.devices, + availableDevices: action.devices, + currentSelectedId: state.selectedDeviceId, + preferredDeviceId: state.preferredDeviceId, + }); + + return { + ...state, + devices: action.devices, + selectedDeviceId: chosenDevice, + }; + } + case "selectDevice": { + const chosenDevice = this.chooseEffectiveSelection({ + previousDevices: state.devices, + availableDevices: state.devices, + currentSelectedId: state.selectedDeviceId, + preferredDeviceId: action.deviceId, + }); + + return { + ...state, + preferredDeviceId: action.deviceId, + selectedDeviceId: chosenDevice, + }; + } + } + }, initialState), + ), + ); + } + + private effectiveSelectionFromState$( + state$: Observable, + ): Behavior { + return this.scope.behavior( + state$ + .pipe( + map((state) => { + if (state.selectedDeviceId) { + return { + id: state.selectedDeviceId, + /** This is an iOS thing, always false for android*/ + virtualEarpiece: false, + }; + } + return undefined; + }), + distinctUntilChanged((a, b) => a?.id === b?.id), + ) + .pipe( + tap((selected) => { + this.logger.debug(`selected device: ${selected?.id}`); + }), + ), + ); + } + + private chooseEffectiveSelection(args: { + previousDevices: OutputDevice[]; + availableDevices: OutputDevice[]; + currentSelectedId: string | undefined; + preferredDeviceId: string | undefined; + }): string | undefined { + const { + previousDevices, + availableDevices, + currentSelectedId, + preferredDeviceId, + } = args; + + this.logger.debug(`chooseEffectiveSelection with args:`, args); + + // Take preferredDeviceId in priority or default to the last effective selection. + const activeSelectedDeviceId = preferredDeviceId || currentSelectedId; + const isAvailable = availableDevices.some( + (device) => device.id === activeSelectedDeviceId, + ); + + // If there is no current device, or it is not available anymore, + // choose the default device selection logic. + if (activeSelectedDeviceId === undefined || !isAvailable) { + this.logger.debug( + `No current device or it is not available, using default selection logic.`, + ); + // use the default selection logic + return this.chooseDefaultDeviceId(availableDevices); + } + + // Is there a new added device? + // If a device is added, we might want to switch to it if it's more preferred than the currently selected device. + const newDeviceWasAdded = availableDevices.some( + (device) => !previousDevices.some((d) => d.id === device.id), + ); + + if (newDeviceWasAdded) { + // TODO only want to check from the added device, not all devices.? + // check if the currently selected device is the most preferred one, if not switch to the most preferred one. + const mostPreferredDevice = availableDevices[0]; + this.logger.debug( + `A new device was added, checking if we should switch to it.`, + mostPreferredDevice, + ); + if (mostPreferredDevice.id !== activeSelectedDeviceId) { + // Given this is automatic switching, we want to be careful and only switch to a more private device + // (e.g. from speaker to a BT headset) but not switch from a more private device to a less private one + // (e.g. from a BT headset to the speaker), as that can be disruptive for the user if it happens unexpectedly. + if (mostPreferredDevice.isExternalHeadset == true) { + this.logger.info( + `The currently selected device ${mostPreferredDevice.id} is not the most preferred one, switching to the most preferred one ${activeSelectedDeviceId} instead.`, + ); + // Let's switch as it is a more private device. + return mostPreferredDevice.id; + } + } + } + + // no changes + return activeSelectedDeviceId; + } + + /** + * The logic for the default is different based on the call type. + * For example for a voice call we want to default to the earpiece if it's available, + * but for a video call we want to default to the speaker. + * If the user is using a BT headset we want to default to that, as it's likely what they want to use for both video and voice calls. + * + * @param available the available audio output devices to choose from, keyed by their id, sorted by likelihood of it being used for communication. + * + */ + private chooseDefaultDeviceId(available: OutputDevice[]): string | undefined { + this.logger.debug( + `Android routing logic intent: ${this.initialIntent} finding best default...`, + ); + if (this.initialIntent === "audio") { + const systemProposed = available[0]; + // If no headset is connected, android will route to the speaker by default, + // but for a voice call we want to route to the earpiece instead, + // so override the system proposed routing in that case. + if (systemProposed?.isSpeaker == true) { + // search for the earpiece + const earpieceDevice = available.find( + (device) => device.isEarpiece == true, + ); + if (earpieceDevice) { + this.logger.debug( + `Android routing: Switch to earpiece instead of speaker for voice call`, + ); + return earpieceDevice.id; + } else { + this.logger.debug( + `Android routing: no earpiece found, cannot switch, use system proposed routing`, + ); + return systemProposed.id; + } + } else { + this.logger.debug( + `Android routing: Use system proposed routing `, + systemProposed, + ); + return systemProposed?.id; + } + } else { + // Use the system best proposed best routing. + return available[0]?.id; + } + } +} + +// Utilities +function mapDeviceToLabel(device: OutputDevice): AudioOutputDeviceLabel { + const { name, isEarpiece, isSpeaker } = device; + if (isEarpiece) return { type: "earpiece" }; + else if (isSpeaker) return { type: "speaker" }; + else return { type: "name", name }; +} diff --git a/src/state/MediaDevices.ts b/src/state/MediaDevices.ts index cea97519..cf578fb4 100644 --- a/src/state/MediaDevices.ts +++ b/src/state/MediaDevices.ts @@ -15,6 +15,7 @@ import { Subject, switchMap, type Observable, + tap, } from "rxjs"; import { createMediaDeviceObserver } from "@livekit/components-core"; import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; @@ -34,6 +35,7 @@ import { getUrlParams } from "../UrlParams"; import { platform } from "../Platform"; import { switchWhen } from "../utils/observable"; import { type Behavior, constant } from "./Behavior"; +import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts"; // This hardcoded id is used in EX ios! It can only be changed in coordination with // the ios swift team. @@ -49,10 +51,18 @@ export type AudioOutputDeviceLabel = | { type: "earpiece" } | { type: "default"; name: string | null }; +/** + * Base selected-device value shared by all media kinds. + * + * `id` is the effective device identifier used by browser media APIs. + */ export interface SelectedDevice { id: string; } +/** + * Selected audio input value with audio-input-specific metadata. + */ export interface SelectedAudioInputDevice extends SelectedDevice { /** * Emits whenever we think that this audio input device has logically changed @@ -61,6 +71,9 @@ export interface SelectedAudioInputDevice extends SelectedDevice { hardwareDeviceChange$: Observable; } +/** + * Selected audio output value with output-routing-specific metadata. + */ export interface SelectedAudioOutputDevice extends SelectedDevice { /** * Whether this device is a "virtual earpiece" device. If so, we should output @@ -69,23 +82,42 @@ export interface SelectedAudioOutputDevice extends SelectedDevice { virtualEarpiece: boolean; } +/** + * Common reactive contract for selectable input/output media devices (mic, speaker, camera). + * + * `Label` is the type used to represent a device in UI lists. + * `Selected` is the type used to represent the active selection for a device kind. + */ export interface MediaDevice { /** - * A map from available device IDs to labels. + * Reactive map of currently available devices keyed by device ID. + * + * `Label` defines the UI-facing label data structure for each device type. */ available$: Behavior>; + /** - * The selected device. + * The active device selection. + * Can be `undefined` when no device is yet selected. + * + * When defined, `Selected` contains the selected device ID plus any + * type-specific metadata. */ selected$: Behavior; + /** - * Selects a new device. + * Requests selection of a device by ID. + * + * Implementations typically persist this preference and let `selected$` + * converge to the effective device (which may differ if the requested ID is + * unavailable). */ select(id: string): void; } /** * An observable that represents if we should display the devices menu for iOS. + * * This implies the following * - hide any input devices (they do not work anyhow on ios) * - Show a button to show the native output picker instead. @@ -143,19 +175,29 @@ function buildDeviceMap( function selectDevice$