studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Earpiece switcher and overlay (#3347)

Browse Source 
* Add a global control for toggling earpiece mode

This will be used by Element X to show an earpiece toggle button in the header.

* Add an earpiece overlay


* Fix header
The header needs to be passed forward as a string to some components and as a bool (hideHeader) to others.
Also use a enum instead of string options.

* fix top clipping with header


* hide app bar in pip

* revert android overlay app_bar

* Modernize AppBarContext

* Style header icon color as desired and switch earpice/speaker icon

* fix initial selection when using controlled media

* Add "Back to video" button

* fix tests

* remove dead code

* add snapshot test

* fix back to video button

* Request capability to learn the room name

We now need the room name in order to implement the mobile (widget-based) designs with the app bar.

* Test the CallViewModel output switcher directly

---------

Co-authored-by: Timo <toger5@hotmail.de>
This commit is contained in:
Robin
2025-06-26 05:08:57 -04:00
committed by GitHub
parent c012aec909
commit f509c06cc6
33 changed files with 942 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/state/CallViewModel.test.ts
View File

@@ -42,6 +42,7 @@ import {
withTestScheduler,
mockRtcMembership,
MockRTCSession,
mockMediaDevices,
} from "../utils/test";
import {
ECAddonConnectionState,
@@ -71,6 +72,12 @@ import {
localId,
localRtcMember,
} from "../utils/test-fixtures";
import { ObservableScope } from "./ObservableScope";
import { MediaDevices } from "./MediaDevices";
import { getValue } from "../utils/observable";
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
vi.mock("@livekit/components-core");
@@ -209,6 +216,7 @@ function withCallViewModel(
rtcMembers$: Observable<Partial<CallMembership>[]>,
connectionState$: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
mediaDevices: MediaDevices,
continuation: (
vm: CallViewModel,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
@@ -262,6 +270,7 @@ function withCallViewModel(
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
mediaDevices,
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -301,6 +310,7 @@ test("participants are retained during a focus switch", () => {
s: ECAddonConnectionState.ECSwitchingFocus,
}),
new Map(),
mockMediaDevices({}),
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
@@ -340,6 +350,7 @@ test("screen sharing activates spotlight layout", () => {
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
@@ -423,6 +434,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
[bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })],
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
mockMediaDevices({}),
(vm) => {
schedule(visibilityInputMarbles, {
a: () => {
@@ -479,6 +491,7 @@ test("participants adjust order when space becomes constrained", () => {
[bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })],
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
mockMediaDevices({}),
(vm) => {
let setVisibleTiles: ((value: number) => void) | null = null;
vm.layout$.subscribe((layout) => {
@@ -532,6 +545,7 @@ test("spotlight speakers swap places", () => {
[bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })],
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
mockMediaDevices({}),
(vm) => {
schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") });
@@ -577,6 +591,7 @@ test("layout enters picture-in-picture mode when requested", () => {
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
schedule(pipControlInputMarbles, {
e: () => window.controls.enablePip(),
@@ -618,6 +633,7 @@ test("spotlight remembers whether it's expanded", () => {
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
@@ -686,6 +702,7 @@ test("participants must have a MatrixRTCSession to be visible", () => {
}),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -730,6 +747,7 @@ test("shows participants without MatrixRTCSession when enabled in settings", ()
of([]), // No one joins the MatrixRTC session
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -777,6 +795,7 @@ it("should show at least one tile per MatrixRTCSession", () => {
}),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -825,6 +844,7 @@ test("should disambiguate users with the same displayname", () => {
}),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
@@ -875,6 +895,7 @@ test("should disambiguate users with invisible characters", () => {
}),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
@@ -911,6 +932,7 @@ test("should strip RTL characters from displayname", () => {
}),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
@@ -943,6 +965,7 @@ it("should rank raised hands above video feeds and below speakers and presenters
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
mockMediaDevices({}),
(vm, { raisedHands$ }) => {
schedule("ab", {
a: () => {
@@ -991,3 +1014,47 @@ it("should rank raised hands above video feeds and below speakers and presenters
);
});
});
test("audio output changes when toggling earpiece mode", () => {
withTestScheduler(({ schedule, expectObservable }) => {
getUrlParams.mockReturnValue({ controlledAudioDevices: true });
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(of([]));
const scope = new ObservableScope();
onTestFinished(() => scope.end());
const devices = new MediaDevices(scope);
window.controls.setAvailableAudioDevices([
{ id: "speaker", name: "Speaker", isSpeaker: true },
{ id: "earpiece", name: "Earpiece", isEarpiece: true },
{ id: "headphones", name: "Headphones" },
]);
window.controls.setAudioDevice("headphones");
const toggleInputMarbles = " -aaa";
const expectedEarpieceModeMarbles = "n-yn";
const expectedTargetStateMarbles = " sese";
withCallViewModel(
of([]),
of([]),
of(ConnectionState.Connected),
new Map(),
devices,
(vm) => {
schedule(toggleInputMarbles, {
a: () => getValue(vm.audioOutputSwitcher$)?.switch(),
});
expectObservable(vm.earpieceMode$).toBe(expectedEarpieceModeMarbles, {
n: false,
y: true,
});
expectObservable(
vm.audioOutputSwitcher$.pipe(
map((switcher) => switcher?.targetOutput),
),
).toBe(expectedTargetStateMarbles, { s: "speaker", e: "earpiece" });
},
);
});
});

47
src/state/CallViewModel.ts
View File

@@ -93,6 +93,7 @@ import {
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices";
// How long we wait after a focus switch before showing the real participant
// list again
@@ -1246,6 +1247,51 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
/**
* Whether audio is currently being output through the earpiece.
*/
public readonly earpieceMode$: Observable<boolean> = combineLatest(
[
this.mediaDevices.audioOutput.available$,
this.mediaDevices.audioOutput.selected$,
],
(available, selected) =>
selected !== undefined && available.get(selected.id)?.type === "earpiece",
).pipe(this.scope.state());
/**
* Callback to toggle between the earpiece and the loudspeaker.
*
* This will be `null` in case the target does not exist in the list
* of available audio outputs.
*/
public readonly audioOutputSwitcher$: Observable<{
targetOutput: "earpiece" | "speaker";
switch: () => void;
} | null> = combineLatest(
[
this.mediaDevices.audioOutput.available$,
this.mediaDevices.audioOutput.selected$,
],
(available, selected) => {
const selectionType = selected && available.get(selected.id)?.type;
// If we are in any output mode other than spaeker switch to speaker.
const newSelectionType =
selectionType === "speaker" ? "earpiece" : "speaker";
const newSelection = [...available].find(
([, d]) => d.type === newSelectionType,
);
if (newSelection === undefined) return null;
const [id] = newSelection;
return {
targetOutput: newSelectionType,
switch: () => this.mediaDevices.audioOutput.select(id),
};
},
);
public readonly reactions$ = this.reactionsSubject$.pipe(
map((v) =>
Object.fromEntries(
@@ -1336,6 +1382,7 @@ export class CallViewModel extends ViewModel {
// A call is permanently tied to a single Matrix room and LiveKit room
private readonly matrixRTCSession: MatrixRTCSession,
private readonly livekitRoom: LivekitRoom,
private readonly mediaDevices: MediaDevices,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState$: Observable<ECConnectionState>,
private readonly handsRaisedSubject$: Observable<

14
src/state/MediaDevices.ts
View File

@@ -42,10 +42,13 @@ const logger = rootLogger.getChild("[MediaDevices]");
export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number }
| { type: "default"; name: string | null };
| { type: "number"; number: number };
export type AudioOutputDeviceLabel = DeviceLabel | { type: "earpiece" };
export type AudioOutputDeviceLabel =
| DeviceLabel
| { type: "speaker" }
| { type: "earpiece" }
| { type: "default"; name: string | null };
export interface SelectedDevice {
id: string;
@@ -211,7 +214,8 @@ class AudioOutput
this.scope,
).pipe(
map((availableRaw) => {
const available = buildDeviceMap(availableRaw);
const available: Map<string, AudioOutputDeviceLabel> =
buildDeviceMap(availableRaw);
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
@@ -269,7 +273,7 @@ class ControlledAudioOutput
let deviceLabel: AudioOutputDeviceLabel;
// if (isExternalHeadset) // Do we want this?
if (isEarpiece) deviceLabel = { type: "earpiece" };
else if (isSpeaker) deviceLabel = { type: "default", name };
else if (isSpeaker) deviceLabel = { type: "speaker" };
else deviceLabel = { type: "name", name };
return [id, deviceLabel];
},