Add camera switching to the media view model

src/state/CallViewModel.ts

@@ -255,6 +255,7 @@ class UserMedia {
     participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    mediaDevices: MediaDevices,
     displayname$: Observable<string>,
     handRaised$: Observable<Date | null>,
     reaction$: Observable<ReactionOption | null>,
@@ -268,6 +269,7 @@ class UserMedia {
         this.participant$.asObservable() as Observable<LocalParticipant>,
         encryptionSystem,
         livekitRoom,
+        mediaDevices,
         displayname$,
         handRaised$,
         reaction$,
@@ -565,6 +567,7 @@ export class CallViewModel extends ViewModel {
                       participant,
                       this.encryptionSystem,
                       this.livekitRoom,
+                      this.mediaDevices,
                       this.memberDisplaynames$.pipe(
                         map((m) => m.get(matrixIdentifier) ?? "[👻]"),
                       ),
@@ -629,6 +632,7 @@ export class CallViewModel extends ViewModel {
                             participant,
                             this.encryptionSystem,
                             this.livekitRoom,
+                            this.mediaDevices,
                             this.memberDisplaynames$.pipe(
                               map((m) => m.get(participant.identity) ?? "[👻]"),
                             ),

src/state/MediaViewModel.test.ts

@@ -5,14 +5,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { expect, test, vi } from "vitest";
+import { expect, onTestFinished, test, vi } from "vitest";
+import { of } from "rxjs";
+import {
+  type LocalTrackPublication,
+  LocalVideoTrack,
+  TrackEvent,
+} from "livekit-client";
+import { waitFor } from "@testing-library/dom";
 
 import {
+  mockLocalParticipant,
+  mockMediaDevices,
   mockRtcMembership,
   withLocalMedia,
   withRemoteMedia,
   withTestScheduler,
 } from "../utils/test";
+import { getValue } from "../utils/observable";
+
+global.MediaStreamTrack = class {} as unknown as {
+  new (): MediaStreamTrack;
+  prototype: MediaStreamTrack;
+};
+global.MediaStream = class {} as unknown as {
+  new (): MediaStream;
+  prototype: MediaStream;
+};
+
+const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
+vi.mock("../Platform", () => ({
+  get platform(): string {
+    return platformMock();
+  },
+}));
 
 const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
 
@@ -79,17 +105,23 @@ test("toggle fit/contain for a participant's video", async () => {
 });
 
 test("local media remembers whether it should always be shown", async () => {
-  await withLocalMedia(rtcMembership, {}, (vm) =>
+  await withLocalMedia(
-    withTestScheduler(({ expectObservable, schedule }) => {
+    rtcMembership,
-      schedule("-a|", { a: () => vm.setAlwaysShow(false) });
+    {},
-      expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false });
+    mockLocalParticipant({}),
-    }),
+    mockMediaDevices({}),
+    (vm) =>
+      withTestScheduler(({ expectObservable, schedule }) => {
+        schedule("-a|", { a: () => vm.setAlwaysShow(false) });
+        expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false });
+      }),
   );
   // Next local media should start out *not* always shown
   await withLocalMedia(
     rtcMembership,
-
     {},
+    mockLocalParticipant({}),
+    mockMediaDevices({}),
     (vm) =>
       withTestScheduler(({ expectObservable, schedule }) => {
         schedule("-a|", { a: () => vm.setAlwaysShow(true) });
@@ -97,3 +129,76 @@ test("local media remembers whether it should always be shown", async () => {
       }),
   );
 });
+
+test("switch cameras", async () => {
+  // Camera switching is only available on mobile
+  platformMock.mockReturnValue("android");
+  onTestFinished(() => void platformMock.mockReset());
+
+  // Construct a mock video track which knows how to be restarted
+  const track = new LocalVideoTrack({
+    getConstraints() {},
+    addEventListener() {},
+    removeEventListener() {},
+  } as unknown as MediaStreamTrack);
+
+  let deviceId = "front camera";
+  const restartTrack = vi.fn(async ({ facingMode }) => {
+    deviceId = facingMode === "user" ? "front camera" : "back camera";
+    track.emit(TrackEvent.Restarted);
+    return Promise.resolve();
+  });
+  track.restartTrack = restartTrack;
+
+  Object.defineProperty(track, "mediaStreamTrack", {
+    get() {
+      return {
+        label: "Video",
+        getSettings: (): object => ({
+          deviceId,
+          facingMode: deviceId === "front camera" ? "user" : "environment",
+        }),
+      };
+    },
+  });
+
+  const selectVideoInput = vi.fn();
+
+  await withLocalMedia(
+    rtcMembership,
+    {},
+    mockLocalParticipant({
+      getTrackPublication() {
+        return { track } as unknown as LocalTrackPublication;
+      },
+    }),
+    mockMediaDevices({
+      videoInput: {
+        available$: of(new Map()),
+        selected$: of(undefined),
+        select: selectVideoInput,
+      },
+    }),
+    async (vm) => {
+      // Switch to back camera
+      getValue(vm.switchCamera$)!();
+      expect(restartTrack).toHaveBeenCalledTimes(1);
+      expect(restartTrack).toHaveBeenCalledWith({ facingMode: "environment" });
+      await waitFor(() => {
+        expect(selectVideoInput).toHaveBeenCalledTimes(1);
+        expect(selectVideoInput).toHaveBeenCalledWith("back camera");
+      });
+      expect(deviceId).toBe("back camera");
+
+      // Switch to front camera
+      getValue(vm.switchCamera$)!();
+      expect(restartTrack).toHaveBeenCalledTimes(2);
+      expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" });
+      await waitFor(() => {
+        expect(selectVideoInput).toHaveBeenCalledTimes(2);
+        expect(selectVideoInput).toHaveBeenLastCalledWith("front camera");
+      });
+      expect(deviceId).toBe("front camera");
+    },
+  );
+});

src/state/MediaViewModel.ts

@@ -16,6 +16,7 @@ import {
 import {
   type LocalParticipant,
   LocalTrack,
+  LocalVideoTrack,
   type Participant,
   ParticipantEvent,
   type RemoteParticipant,
@@ -27,6 +28,7 @@ import {
   RemoteTrack,
 } from "livekit-client";
 import { type RoomMember } from "matrix-js-sdk";
+import { logger } from "matrix-js-sdk/lib/logger";
 import {
   BehaviorSubject,
   type Observable,
@@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
 import { type ReactionOption } from "../reactions";
+import { platform } from "../Platform";
+import { type MediaDevices } from "./MediaDevices";
 
 export function observeTrackReference$(
   participant$: Observable<Participant | undefined>,
@@ -433,20 +437,35 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
  * The local participant's user media.
  */
 export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
+  /**
+   * The local video track as an observable that emits whenever the track
+   * changes, the camera is switched, or the track is muted.
+   */
+  private readonly videoTrack$: Observable<LocalVideoTrack | null> =
+    this.video$.pipe(
+      switchMap((v) => {
+        const track = v?.publication?.track;
+        if (!(track instanceof LocalVideoTrack)) return of(null);
+        return merge(
+          // Watch for track restarts because they indicate a camera switch
+          fromEvent(track, TrackEvent.Restarted).pipe(
+            startWith(null),
+            map(() => track),
+          ),
+          fromEvent(track, TrackEvent.Muted).pipe(map(() => null)),
+        );
+      }),
+    );
+
   /**
    * Whether the video should be mirrored.
    */
-  public readonly mirror$ = this.video$.pipe(
+  public readonly mirror$ = this.videoTrack$.pipe(
-    switchMap((v) => {
+    // Mirror only front-facing cameras (those that face the user)
-      const track = v?.publication?.track;
+    map(
-      if (!(track instanceof LocalTrack)) return of(false);
+      (track) =>
-      // Watch for track restarts, because they indicate a camera switch
+        track !== null && facingModeFromLocalTrack(track).facingMode === "user",
-      return fromEvent(track, TrackEvent.Restarted).pipe(
+    ),
-        startWith(null),
-        // Mirror only front-facing cameras (those that face the user)
-        map(() => facingModeFromLocalTrack(track).facingMode === "user"),
-      );
-    }),
     this.scope.state(),
   );
 
@@ -457,12 +476,46 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   public readonly alwaysShow$ = alwaysShowSelf.value$;
   public readonly setAlwaysShow = alwaysShowSelf.setValue;
 
+  /**
+   * Callback for switching between the front and back cameras.
+   */
+  public readonly switchCamera$: Observable<(() => void) | null> =
+    platform === "desktop"
+      ? of(null)
+      : this.videoTrack$.pipe(
+          map((track) => {
+            if (track === null) return null;
+            const facingMode = facingModeFromLocalTrack(track).facingMode;
+            // If the camera isn't front or back-facing, don't provide a switch
+            // camera shortcut at all
+            if (facingMode !== "user" && facingMode !== "environment")
+              return null;
+            // Restart the track with a camera facing the opposite direction
+            return (): void =>
+              void track
+                .restartTrack({
+                  facingMode: facingMode === "user" ? "environment" : "user",
+                })
+                .then(() => {
+                  // Inform the MediaDevices which camera was chosen
+                  const deviceId =
+                    track.mediaStreamTrack.getSettings().deviceId;
+                  if (deviceId !== undefined)
+                    this.mediaDevices.videoInput.select(deviceId);
+                })
+                .catch((e) =>
+                  logger.error("Failed to switch camera", facingMode, e),
+                );
+          }),
+        );
+
   public constructor(
     id: string,
     member: RoomMember | undefined,
     participant$: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    private readonly mediaDevices: MediaDevices,
     displayname$: Observable<string>,
     handRaised$: Observable<Date | null>,
     reaction$: Observable<ReactionOption | null>,

src/tile/SpotlightTile.test.tsx

@@ -13,6 +13,8 @@ import { of } from "rxjs";
 
 import { SpotlightTile } from "./SpotlightTile";
 import {
+  mockLocalParticipant,
+  mockMediaDevices,
   mockRtcMembership,
   withLocalMedia,
   withRemoteMedia,
@@ -39,6 +41,8 @@ test("SpotlightTile is accessible", async () => {
           rawDisplayName: "Bob",
           getMxcAvatarUrl: () => "mxc://dlskf",
         },
+        mockLocalParticipant({}),
+        mockMediaDevices({}),
         async (vm2) => {
           const user = userEvent.setup();
           const toggleExpanded = vi.fn();

src/utils/test.ts

@@ -205,9 +205,10 @@ export function mockLocalParticipant(
 export async function withLocalMedia(
   localRtcMember: CallMembership,
   roomMember: Partial<RoomMember>,
+  localParticipant: LocalParticipant,
+  mediaDevices: MediaDevices,
   continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
-  const localParticipant = mockLocalParticipant({});
   const vm = new LocalUserMediaViewModel(
     "local",
     mockMatrixRoomMember(localRtcMember, roomMember),
@@ -216,6 +217,7 @@ export async function withLocalMedia(
       kind: E2eeType.PER_PARTICIPANT,
     },
     mockLivekitRoom({ localParticipant }),
+    mediaDevices,
     of(roomMember.rawDisplayName ?? "nodisplayname"),
     of(null),
     of(null),