studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'livekit' into reconnecting

Browse Source
This commit is contained in:
Robin
2025-08-20 14:13:03 +02:00
parent 5d0289f3c9 25f04030fc
commit a7cc11ac81
21 changed files with 265 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/state/CallViewModel.ts
View File

@@ -14,10 +14,8 @@ import {
ConnectionState,
type Room as LivekitRoom,
type LocalParticipant,
LocalVideoTrack,
ParticipantEvent,
type RemoteParticipant,
Track,
} from "livekit-client";
import {
ClientEvent,
@@ -68,7 +66,6 @@ import {
import {
LocalUserMediaViewModel,
type MediaViewModel,
observeTrackReference$,
RemoteUserMediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
@@ -271,6 +268,7 @@ class UserMedia {
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
mediaDevices: MediaDevices,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
@@ -284,6 +282,7 @@ class UserMedia {
this.participant$ as Behavior<LocalParticipant>,
encryptionSystem,
livekitRoom,
mediaDevices,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
@@ -401,18 +400,6 @@ export class CallViewModel extends ViewModel {
private readonly userId = this.matrixRoom.client.getUserId();
private readonly deviceId = this.matrixRoom.client.getDeviceId();
public readonly localVideo$ = this.scope.behavior<LocalVideoTrack | null>(
observeTrackReference$(
this.livekitRoom.localParticipant,
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
);
/**
* The raw list of RemoteParticipants as reported by LiveKit
*/
@@ -688,6 +675,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
this.livekitRoom,
this.mediaDevices,
this.memberDisplaynames$.pipe(
map((m) => m.get(matrixIdentifier) ?? "[👻]"),
),
@@ -752,6 +740,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
this.livekitRoom,
this.mediaDevices,
this.memberDisplaynames$.pipe(
map(
(m) => m.get(participant.identity) ?? "[👻]",

120
src/state/MediaViewModel.test.ts
View File

@@ -5,14 +5,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test, vi } from "vitest";
import { expect, onTestFinished, test, vi } from "vitest";
import {
type LocalTrackPublication,
LocalVideoTrack,
TrackEvent,
} from "livekit-client";
import { waitFor } from "@testing-library/dom";
import {
mockLocalParticipant,
mockMediaDevices,
mockRtcMembership,
withLocalMedia,
withRemoteMedia,
withTestScheduler,
} from "../utils/test";
import { getValue } from "../utils/observable";
import { constant } from "./Behavior";
global.MediaStreamTrack = class {} as unknown as {
new (): MediaStreamTrack;
prototype: MediaStreamTrack;
};
global.MediaStream = class {} as unknown as {
new (): MediaStream;
prototype: MediaStream;
};
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../Platform", () => ({
get platform(): string {
return platformMock();
},
}));
const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
@@ -79,17 +105,23 @@ test("toggle fit/contain for a participant's video", async () => {
});
test("local media remembers whether it should always be shown", async () => {
await withLocalMedia(rtcMembership, {}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(false) });
expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false });
}),
await withLocalMedia(
rtcMembership,
{},
mockLocalParticipant({}),
mockMediaDevices({}),
(vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(false) });
expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false });
}),
);
// Next local media should start out *not* always shown
await withLocalMedia(
rtcMembership,
{},
mockLocalParticipant({}),
mockMediaDevices({}),
(vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(true) });
@@ -97,3 +129,77 @@ test("local media remembers whether it should always be shown", async () => {
}),
);
});
test("switch cameras", async () => {
// Camera switching is only available on mobile
platformMock.mockReturnValue("android");
onTestFinished(() => void platformMock.mockReset());
// Construct a mock video track which knows how to be restarted
const track = new LocalVideoTrack({
getConstraints() {},
addEventListener() {},
removeEventListener() {},
} as unknown as MediaStreamTrack);
let deviceId = "front camera";
const restartTrack = vi.fn(async ({ facingMode }) => {
deviceId = facingMode === "user" ? "front camera" : "back camera";
track.emit(TrackEvent.Restarted);
return Promise.resolve();
});
track.restartTrack = restartTrack;
Object.defineProperty(track, "mediaStreamTrack", {
get() {
return {
label: "Video",
getSettings: (): object => ({
deviceId,
facingMode: deviceId === "front camera" ? "user" : "environment",
}),
};
},
});
const selectVideoInput = vi.fn();
await withLocalMedia(
rtcMembership,
{},
mockLocalParticipant({
getTrackPublication() {
return { track } as unknown as LocalTrackPublication;
},
}),
mockMediaDevices({
videoInput: {
available$: constant(new Map()),
selected$: constant(undefined),
select: selectVideoInput,
},
}),
async (vm) => {
// Switch to back camera
getValue(vm.switchCamera$)!();
expect(restartTrack).toHaveBeenCalledExactlyOnceWith({
facingMode: "environment",
});
await waitFor(() => {
expect(selectVideoInput).toHaveBeenCalledTimes(1);
expect(selectVideoInput).toHaveBeenCalledWith("back camera");
});
expect(deviceId).toBe("back camera");
// Switch to front camera
getValue(vm.switchCamera$)!();
expect(restartTrack).toHaveBeenCalledTimes(2);
expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" });
await waitFor(() => {
expect(selectVideoInput).toHaveBeenCalledTimes(2);
expect(selectVideoInput).toHaveBeenLastCalledWith("front camera");
});
expect(deviceId).toBe("front camera");
},
);
});

74
src/state/MediaViewModel.ts
View File

@@ -16,6 +16,7 @@ import {
import {
type LocalParticipant,
LocalTrack,
LocalVideoTrack,
type Participant,
ParticipantEvent,
type RemoteParticipant,
@@ -27,6 +28,7 @@ import {
RemoteTrack,
} from "livekit-client";
import { type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
type Observable,
@@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { type ReactionOption } from "../reactions";
import { platform } from "../Platform";
import { type MediaDevices } from "./MediaDevices";
import { type Behavior } from "./Behavior";
export function observeTrackReference$(
@@ -443,20 +447,38 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
*/
export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
/**
* Whether the video should be mirrored.
* The local video track as an observable that emits whenever the track
* changes, the camera is switched, or the track is muted.
*/
public readonly mirror$ = this.scope.behavior(
private readonly videoTrack$: Observable<LocalVideoTrack | null> =
this.video$.pipe(
switchMap((v) => {
const track = v?.publication?.track;
if (!(track instanceof LocalTrack)) return of(false);
// Watch for track restarts, because they indicate a camera switch
return fromEvent(track, TrackEvent.Restarted).pipe(
startWith(null),
// Mirror only front-facing cameras (those that face the user)
map(() => facingModeFromLocalTrack(track).facingMode === "user"),
if (!(track instanceof LocalVideoTrack)) return of(null);
return merge(
// Watch for track restarts because they indicate a camera switch.
// This event is also emitted when unmuting the track object.
fromEvent(track, TrackEvent.Restarted).pipe(
startWith(null),
map(() => track),
),
// When the track object is muted, reset it to null.
fromEvent(track, TrackEvent.Muted).pipe(map(() => null)),
);
}),
);
/**
* Whether the video should be mirrored.
*/
public readonly mirror$ = this.scope.behavior(
this.videoTrack$.pipe(
// Mirror only front-facing cameras (those that face the user)
map(
(track) =>
track !== null &&
facingModeFromLocalTrack(track).facingMode === "user",
),
),
);
@@ -467,12 +489,48 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
public readonly alwaysShow$ = alwaysShowSelf.value$;
public readonly setAlwaysShow = alwaysShowSelf.setValue;
/**
* Callback for switching between the front and back cameras.
*/
public readonly switchCamera$: Behavior<(() => void) | null> =
this.scope.behavior(
platform === "desktop"
? of(null)
: this.videoTrack$.pipe(
map((track) => {
if (track === null) return null;
const facingMode = facingModeFromLocalTrack(track).facingMode;
// If the camera isn't front or back-facing, don't provide a switch
// camera shortcut at all
if (facingMode !== "user" && facingMode !== "environment")
return null;
// Restart the track with a camera facing the opposite direction
return (): void =>
void track
.restartTrack({
facingMode: facingMode === "user" ? "environment" : "user",
})
.then(() => {
// Inform the MediaDevices which camera was chosen
const deviceId =
track.mediaStreamTrack.getSettings().deviceId;
if (deviceId !== undefined)
this.mediaDevices.videoInput.select(deviceId);
})
.catch((e) =>
logger.error("Failed to switch camera", facingMode, e),
);
}),
),
);
public constructor(
id: string,
member: RoomMember | undefined,
participant$: Behavior<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
private readonly mediaDevices: MediaDevices,
displayName$: Behavior<string>,
handRaised$: Behavior<Date | null>,
reaction$: Behavior<ReactionOption | null>,