Merge branch 'livekit' into toger5/lib-ec-version

package.json

@@ -56,7 +56,7 @@
     "@opentelemetry/sdk-trace-base": "^2.0.0",
     "@opentelemetry/sdk-trace-web": "^2.0.0",
     "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@playwright/test": "^1.56.1",
+    "@playwright/test": "^1.57.0",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-slider": "^1.1.2",
     "@radix-ui/react-visually-hidden": "^1.0.3",

playwright/fixtures/widget-user.ts

@@ -111,19 +111,27 @@ async function registerUser(
   await page.getByRole("textbox", { name: "Confirm password" }).click();
   await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD);
   await page.getByRole("button", { name: "Register" }).click();
-  const continueButton = page.getByRole("button", { name: "Continue" });
+
-  try {
-    await expect(continueButton).toBeVisible({ timeout: 5000 });
-    await page
-      .getByRole("textbox", { name: "Password", exact: true })
-      .fill(PASSWORD);
-    await continueButton.click();
-  } catch {
-    // continueButton not visible, continue as normal
-  }
   await expect(
     page.getByRole("heading", { name: `Welcome ${username}` }),
   ).toBeVisible();
+
+  const browserUnsupportedToast = page
+    .getByText("Element does not support this browser")
+    .locator("..")
+    .locator("..");
+
+  // Dismiss incompatible browser toast
+  const dismissButton = browserUnsupportedToast.getByRole("button", {
+    name: "Dismiss",
+  });
+  try {
+    await expect(dismissButton).toBeVisible({ timeout: 700 });
+    await dismissButton.click();
+  } catch {
+    // dismissButton not visible, continue as normal
+  }
+
   await setDevToolElementCallDevUrl(page);
 
   const clientHandle = await page.evaluateHandle(() =>

src/room/GroupCallView.tsx

@@ -160,6 +160,7 @@ export const GroupCallView: FC<Props> = ({
   }, [rtcSession]);
 
   // TODO move this into the callViewModel LocalMembership.ts
+  // We might actually not need this at all. Since we get into fatalError on those errors already?
   useTypedEventEmitter(
     rtcSession,
     MatrixRTCSessionEvent.MembershipManagerError,
@@ -313,6 +314,7 @@ export const GroupCallView: FC<Props> = ({
 
   const navigate = useNavigate();
 
+  // TODO split this into leave and onDisconnect
   const onLeft = useCallback(
     (
       reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",

src/room/InCallView.tsx

@@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
 import classNames from "classnames";
 import { BehaviorSubject, map } from "rxjs";
 import { useObservable } from "observable-hooks";
-import { logger } from "matrix-js-sdk/lib/logger";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import {
   VoiceCallSolidIcon,
   VolumeOnSolidIcon,
@@ -88,6 +88,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
 import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 import {
   debugTileLayout as debugTileLayoutSetting,
+  matrixRTCMode as matrixRTCModeSetting,
   useSetting,
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
@@ -109,6 +110,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t
 import { type Layout } from "../state/layout-types.ts";
 import { ObservableScope } from "../state/ObservableScope.ts";
 
+const logger = rootLogger.getChild("[InCallView]");
+
 const maxTapDurationMs = 400;
 
 export interface ActiveCallProps
@@ -127,6 +130,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const mediaDevices = useMediaDevices();
   const trackProcessorState$ = useTrackProcessorObservable$();
   useEffect(() => {
+    logger.info("START CALL VIEW SCOPE");
     const scope = new ObservableScope();
     const reactionsReader = new ReactionsReader(scope, props.rtcSession);
     const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
@@ -141,6 +145,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
         encryptionSystem: props.e2eeSystem,
         autoLeaveWhenOthersLeft,
         waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
+        matrixRTCMode$: matrixRTCModeSetting.value$,
       },
       reactionsReader.raisedHands$,
       reactionsReader.reactions$,
@@ -151,7 +156,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     setVm(vm);
 
     vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
+
     return (): void => {
+      logger.info("END CALL VIEW SCOPE");
       scope.end();
     };
   }, [
@@ -270,7 +277,10 @@ export const InCallView: FC<InCallViewProps> = ({
   const ringOverlay = useBehavior(vm.ringOverlay$);
   const fatalCallError = useBehavior(vm.fatalError$);
   // Stop the rendering and throw for the error boundary
-  if (fatalCallError) throw fatalCallError;
+  if (fatalCallError) {
+    logger.debug("fatalCallError stop rendering", fatalCallError);
+    throw fatalCallError;
+  }
 
   // We need to set the proper timings on the animation based upon the sound length.
   const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;

src/room/LobbyView.tsx

@@ -79,9 +79,9 @@ export const LobbyView: FC<Props> = ({
   waitingForInvite,
 }) => {
   useEffect(() => {
-    logger.info("[Lifecycle] GroupCallView Component mounted");
+    logger.info("[Lifecycle] LobbyView Component mounted");
     return (): void => {
-      logger.info("[Lifecycle] GroupCallView Component unmounted");
+      logger.info("[Lifecycle] LobbyView Component unmounted");
     };
   }, []);
 

src/state/CallViewModel/CallViewModel.test.ts

@@ -60,7 +60,8 @@ import {
 import { MediaDevices } from "../MediaDevices.ts";
 import { getValue } from "../../utils/observable.ts";
 import { type Behavior, constant } from "../Behavior.ts";
-import { withCallViewModel } from "./CallViewModelTestUtils.ts";
+import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts";
+import { MatrixRTCMode } from "../../settings/settings.ts";
 
 vi.mock("rxjs", async (importOriginal) => ({
   ...(await importOriginal()),
@@ -229,7 +230,13 @@ function mockRingEvent(
 // need a value to fill in for them when emitting notifications
 const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
 
-describe("CallViewModel", () => {
+describe.each([
+  [MatrixRTCMode.Legacy],
+  [MatrixRTCMode.Compatibil],
+  [MatrixRTCMode.Matrix_2_0],
+])("CallViewModel (%s mode)", (mode) => {
+  const withCallViewModel = withCallViewModelInMode(mode);
+
   test("participants are retained during a focus switch", () => {
     withTestScheduler(({ behavior, expectObservable }) => {
       // Participants disappear on frame 2 and come back on frame 3

src/state/CallViewModel/CallViewModel.ts

@@ -16,6 +16,7 @@ import {
 } from "livekit-client";
 import { type Room as MatrixRoom } from "matrix-js-sdk";
 import {
+  catchError,
   combineLatest,
   distinctUntilChanged,
   filter,
@@ -53,11 +54,15 @@ import {
   ScreenShareViewModel,
   type UserMediaViewModel,
 } from "../MediaViewModel";
-import { accumulate, generateItems, pauseWhen } from "../../utils/observable";
+import {
+  accumulate,
+  filterBehavior,
+  generateItems,
+  pauseWhen,
+} from "../../utils/observable";
 import {
   duplicateTiles,
   MatrixRTCMode,
-  matrixRTCMode,
   playReactionsSound,
   showReactions,
 } from "../../settings/settings";
@@ -94,7 +99,7 @@ import {
   type SpotlightLandscapeLayoutMedia,
   type SpotlightPortraitLayoutMedia,
 } from "../layout-types.ts";
-import { type ElementCallError } from "../../utils/errors.ts";
+import { ElementCallError } from "../../utils/errors.ts";
 import { type ObservableScope } from "../ObservableScope.ts";
 import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
 import {
@@ -112,7 +117,8 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
 import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
 import {
   createMatrixLivekitMembers$,
-  type MatrixLivekitMember,
+  type TaggedParticipant,
+  type LocalMatrixLivekitMember,
 } from "./remoteMembers/MatrixLivekitMembers.ts";
 import {
   type AutoLeaveReason,
@@ -151,6 +157,8 @@ export interface CallViewModelOptions {
   connectionState$?: Behavior<ConnectionState>;
   /** Optional behavior overriding the computed window size, mainly for testing purposes. */
   windowSize$?: Behavior<{ width: number; height: number }>;
+  /** The version & compatibility mode of MatrixRTC that we should use. */
+  matrixRTCMode$: Behavior<MatrixRTCMode>;
 }
 
 // Do not play any sounds if the participant count has exceeded this
@@ -430,7 +438,7 @@ export function createCallViewModel$(
     client,
     roomId: matrixRoom.roomId,
     useOldestMember$: scope.behavior(
-      matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
+      options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
     ),
   });
 
@@ -450,7 +458,18 @@ export function createCallViewModel$(
     connectionFactory: connectionFactory,
     inputTransports$: scope.behavior(
       combineLatest(
-        [localTransport$, membershipsAndTransports.transports$],
+        [
+          localTransport$.pipe(
+            catchError((e: unknown) => {
+              logger.info(
+                "dont pass local transport to createConnectionManager$. localTransport$ threw an error",
+                e,
+              );
+              return of(null);
+            }),
+          ),
+          membershipsAndTransports.transports$,
+        ],
         (localTransport, transports) => {
           const localTransportAsArray = localTransport ? [localTransport] : [];
           return transports.mapInner((transports) => [
@@ -471,7 +490,7 @@ export function createCallViewModel$(
   });
 
   const connectOptions$ = scope.behavior(
-    matrixRTCMode.value$.pipe(
+    options.matrixRTCMode$.pipe(
       map((mode) => ({
         encryptMedia: livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each change of matrixRTCMode...
@@ -482,13 +501,13 @@ export function createCallViewModel$(
 
   const localMembership = createLocalMembership$({
     scope: scope,
-    homeserverConnected$: createHomeserverConnected$(
+    homeserverConnected: createHomeserverConnected$(
       scope,
       client,
       matrixRTCSession,
     ),
     muteStates: muteStates,
-    joinMatrixRTC: async (transport: LivekitTransport) => {
+    joinMatrixRTC: (transport: LivekitTransport) => {
       return enterRTCSession(
         matrixRTCSession,
         transport,
@@ -525,22 +544,21 @@ export function createCallViewModel$(
     ),
   );
 
-  const localMatrixLivekitMemberUninitialized = {
-    membership$: localRtcMembership$,
-    participant$: localMembership.participant$,
-    connection$: localMembership.connection$,
-    userId: userId,
-  };
-
   const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
     scope.behavior(
       localRtcMembership$.pipe(
-        switchMap((membership) => {
+        filterBehavior((membership) => membership !== null),
-          if (!membership) return of(null);
+        map((membership$) => {
-          return of(
+          if (membership$ === null) return null;
-            // casting is save here since we know that localRtcMembership$ is !== null since we reached this case.
+          return {
-            localMatrixLivekitMemberUninitialized as LocalMatrixLivekitMember,
+            membership$,
-          );
+            participant: {
+              type: "local" as const,
+              value$: localMembership.participant$,
+            },
+            connection$: localMembership.connection$,
+            userId,
+          };
         }),
       ),
     );
@@ -612,7 +630,7 @@ export function createCallViewModel$(
         const members = membersWithEpoch.value;
         const a$ = combineLatest(
           members.map((member) =>
-            combineLatest([member.connection$, member.participant$]).pipe(
+            combineLatest([member.connection$, member.participant.value$]).pipe(
               map(([connection, participant]) => {
                 // do not render audio for local participant
                 if (!connection || !participant || participant.isLocal)
@@ -696,7 +714,7 @@ export function createCallViewModel$(
           let localParticipantId: string | undefined = undefined;
           // add local member if available
           if (localMatrixLivekitMember) {
-            const { userId, participant$, connection$, membership$ } =
+            const { userId, participant, connection$, membership$ } =
               localMatrixLivekitMember;
             localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
             // const participantId = membership$.value.membershipID;
@@ -707,7 +725,7 @@ export function createCallViewModel$(
                     dup,
                     localParticipantId,
                     userId,
-                    participant$,
+                    participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
                     connection$,
                   ],
                   data: undefined,
@@ -718,7 +736,7 @@ export function createCallViewModel$(
           // add remote members that are available
           for (const {
             userId,
-            participant$,
+            participant,
             connection$,
             membership$,
           } of matrixLivekitMembers) {
@@ -727,7 +745,7 @@ export function createCallViewModel$(
             // const participantId = membership$.value?.identity;
             for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
               yield {
-                keys: [dup, participantId, userId, participant$, connection$],
+                keys: [dup, participantId, userId, participant, connection$],
                 data: undefined,
               };
             }
@@ -739,7 +757,7 @@ export function createCallViewModel$(
           dup,
           participantId,
           userId,
-          participant$,
+          participant,
           connection$,
         ) => {
           const livekitRoom$ = scope.behavior(
@@ -758,7 +776,7 @@ export function createCallViewModel$(
             scope,
             `${participantId}:${dup}`,
             userId,
-            participant$,
+            participant,
             options.encryptionSystem,
             livekitRoom$,
             focusUrl$,
@@ -968,11 +986,12 @@ export function createCallViewModel$(
     ),
   );
 
-  const hasRemoteScreenShares$: Observable<boolean> = spotlight$.pipe(
+  const hasRemoteScreenShares$ = scope.behavior<boolean>(
+    spotlight$.pipe(
       map((spotlight) =>
         spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
       ),
-    distinctUntilChanged(),
+    ),
   );
 
   const pipEnabled$ = scope.behavior(setPipEnabled$, false);
@@ -1446,15 +1465,46 @@ export function createCallViewModel$(
   // reassigned here to make it publicly accessible
   const toggleScreenSharing = localMembership.toggleScreenSharing;
 
+  const errors$ = scope.behavior<{
+    transportError?: ElementCallError;
+    matrixError?: ElementCallError;
+    connectionError?: ElementCallError;
+    publishError?: ElementCallError;
+  } | null>(
+    localMembership.localMemberState$.pipe(
+      map((value) => {
+        const returnObject: {
+          transportError?: ElementCallError;
+          matrixError?: ElementCallError;
+          connectionError?: ElementCallError;
+          publishError?: ElementCallError;
+        } = {};
+        if (value instanceof ElementCallError) return { transportError: value };
+        if (value === TransportState.Waiting) return null;
+        if (value.matrix instanceof ElementCallError)
+          returnObject.matrixError = value.matrix;
+        if (value.media instanceof ElementCallError)
+          returnObject.publishError = value.media;
+        else if (
+          typeof value.media === "object" &&
+          value.media.connection instanceof ElementCallError
+        )
+          returnObject.connectionError = value.media.connection;
+        return returnObject;
+      }),
+    ),
+    null,
+  );
+
   return {
     autoLeave$,
     callPickupState$,
     ringOverlay$,
     leave$,
     hangup: (): void => userHangup$.next(),
-    join: localMembership.requestConnect,
+    join: localMembership.requestJoinAndPublish,
-    toggleScreenSharing,
+    toggleScreenSharing: toggleScreenSharing,
-    sharingScreen$,
+    sharingScreen$: sharingScreen$,
 
     tapScreen: (): void => screenTap$.next(),
     tapControls: (): void => controlsTap$.next(),
@@ -1462,9 +1512,17 @@ export function createCallViewModel$(
     unhoverScreen: (): void => screenUnhover$.next(),
 
     fatalError$: scope.behavior(
-      localMembership.connectionState.livekit$.pipe(
+      errors$.pipe(
-        filter((v) => v.state === RTCBackendState.Error),
+        map((errors) => {
-        map((s) => s.error),
+          logger.debug("errors$ to compute any fatal errors:", errors);
+          return (
+            errors?.transportError ??
+            errors?.matrixError ??
+            errors?.connectionError ??
+            null
+          );
+        }),
+        filter((error) => error !== null),
       ),
       null,
     ),
@@ -1480,15 +1538,24 @@ export function createCallViewModel$(
     audibleReactions$,
     visibleReactions$,
 
-    windowMode$,
+    handsRaised$: handsRaised$,
-    spotlightExpanded$,
+    reactions$: reactions$,
-    toggleSpotlightExpanded$,
+    joinSoundEffect$: joinSoundEffect$,
-    gridMode$,
+    leaveSoundEffect$: leaveSoundEffect$,
-    setGridMode,
+    newHandRaised$: newHandRaised$,
-    grid$,
+    newScreenShare$: newScreenShare$,
-    spotlight$,
+    audibleReactions$: audibleReactions$,
-    pip$,
+    visibleReactions$: visibleReactions$,
-    layout$,
+
+    windowMode$: windowMode$,
+    spotlightExpanded$: spotlightExpanded$,
+    toggleSpotlightExpanded$: toggleSpotlightExpanded$,
+    gridMode$: gridMode$,
+    setGridMode: setGridMode,
+    grid$: grid$,
+    spotlight$: spotlight$,
+    pip$: pip$,
+    layout$: layout$,
     userMedia$,
     localMatrixLivekitMember$,
     matrixLivekitMembers$: scope.behavior(
@@ -1499,16 +1566,14 @@ export function createCallViewModel$(
         }),
       ),
     ),
-    tileStoreGeneration$,
+    tileStoreGeneration$: tileStoreGeneration$,
-    showSpotlightIndicators$,
+    showSpotlightIndicators$: showSpotlightIndicators$,
-    showSpeakingIndicators$,
+    showSpeakingIndicators$: showSpeakingIndicators$,
-    showHeader$,
+    showHeader$: showHeader$,
-    showFooter$,
+    showFooter$: showFooter$,
-    earpieceMode$,
+    earpieceMode$: earpieceMode$,
-    audioOutputSwitcher$,
+    audioOutputSwitcher$: audioOutputSwitcher$,
     reconnecting$: localMembership.reconnecting$,
-    connected$: localMembership.connected$,
-    connectionState: localMembership.connectionState,
   };
 }
 

src/state/CallViewModel/CallViewModelTestUtils.ts

@@ -53,6 +53,7 @@ import {
 import { type Behavior, constant } from "../Behavior";
 import { type ProcessorState } from "../../livekit/TrackProcessorContext";
 import { type MediaDevices } from "../MediaDevices";
+import { type MatrixRTCMode } from "../../settings/settings";
 
 mockConfig({
   livekit: { livekit_service_url: "http://my-default-service-url.com" },
@@ -80,7 +81,8 @@ export interface CallViewModelInputs {
 
 const localParticipant = mockLocalParticipant({ identity: "" });
 
-export function withCallViewModel(
+export function withCallViewModel(mode: MatrixRTCMode) {
+  return (
     {
       remoteParticipants$ = constant([]),
       rtcMembers$ = constant([localRtcMember]),
@@ -95,14 +97,13 @@ export function withCallViewModel(
     continuation: (
       vm: CallViewModel,
       rtcSession: MockRTCSession,
-    subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
+      subjects: {
+        raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>>;
+      },
       setSyncState: (value: SyncState) => void,
     ) => void,
-  options: CallViewModelOptions = {
+    options: Partial<CallViewModelOptions> = {},
-    encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+  ): void => {
-    autoLeaveWhenOthersLeft: false,
-  },
-): void {
     let syncState = initialSyncState;
     const setSyncState = (value: SyncState): void => {
       const prev = syncState;
@@ -130,7 +131,9 @@ export function withCallViewModel(
       getMembers: () => Array.from(roomMembers.values()),
       getMembersWithMembership: () => Array.from(roomMembers.values()),
     });
-  const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$);
+    const rtcSession = new MockRTCSession(room, []).withMemberships(
+      rtcMembers$,
+    );
     const participantsSpy = vi
       .spyOn(ComponentsCore, "connectedParticipantsObserver")
       .mockReturnValue(remoteParticipants$);
@@ -157,7 +160,9 @@ export function withCallViewModel(
       .spyOn(ComponentsCore, "roomEventSelector")
       .mockImplementation((_room, _eventType) => of());
     const muteStates = mockMuteStates();
-  const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
+    const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>(
+      {},
+    );
     const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
 
     const vm = createCallViewModel$(
@@ -167,7 +172,8 @@ export function withCallViewModel(
       mediaDevices,
       muteStates,
       {
-      ...options,
+        encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        autoLeaveWhenOthersLeft: false,
         livekitRoomFactory: (): LivekitRoom =>
           mockLivekitRoom({
             localParticipant,
@@ -176,6 +182,8 @@ export function withCallViewModel(
           }),
         connectionState$,
         windowSize$,
+        matrixRTCMode$: constant(mode),
+        ...options,
       },
       raisedHands$,
       reactions$,
@@ -193,4 +201,5 @@ export function withCallViewModel(
     });
 
     continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
+  };
 }

src/state/CallViewModel/LayoutSwitch.test.ts

@@ -5,198 +5,128 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { describe, test } from "vitest";
-import { firstValueFrom, of } from "rxjs";
 
 import { createLayoutModeSwitch } from "./LayoutSwitch";
-import { ObservableScope } from "../ObservableScope";
+import { testScope, withTestScheduler } from "../../utils/test";
-import { constant } from "../Behavior";
-import { withTestScheduler } from "../../utils/test";
 
-let scope: ObservableScope;
+function testLayoutSwitch({
-beforeEach(() => {
+  windowMode = "n",
-  scope = new ObservableScope();
+  hasScreenShares = "n",
-});
+  userSelection = "",
-afterEach(() => {
+  expectedGridMode,
-  scope.end();
+}: {
-});
+  windowMode?: string;
-
+  hasScreenShares?: string;
-describe("Default mode", () => {
+  userSelection?: string;
-  test("Should be in grid layout by default", async () => {
+  expectedGridMode: string;
-    const { gridMode$ } = createLayoutModeSwitch(
+}): void {
-      scope,
+  withTestScheduler(({ behavior, schedule, expectObservable }) => {
-      constant("normal"),
-      of(false),
-    );
-
-    const mode = await firstValueFrom(gridMode$);
-    expect(mode).toBe("grid");
-  });
-
-  test("Should switch to spotlight mode when window mode is flat", async () => {
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      constant("flat"),
-      of(false),
-    );
-
-    const mode = await firstValueFrom(gridMode$);
-    expect(mode).toBe("spotlight");
-  });
-});
-
-test("Should allow switching modes manually", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
     const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
+      testScope(),
-      behavior("n", { n: "normal" }),
+      behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }),
-      cold("f", { f: false, t: true }),
+      behavior(hasScreenShares, { y: true, n: false }),
     );
-
+    schedule(userSelection, {
-    schedule("--sgs", {
-      s: () => setGridMode("spotlight"),
-      g: () => setGridMode("grid"),
-    });
-
-    expectObservable(gridMode$).toBe("g-sgs", {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should switch to spotlight mode when there is a remote screen share", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "f--t";
-    const gridsMarble = "g--s";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Can manually force grid when there is a screenshare", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold("-ft", { f: false, t: true }),
-    );
-
-    schedule("---g", {
-      g: () => setGridMode("grid"),
-    });
-
-    expectObservable(gridMode$).toBe("ggsg", {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should auto-switch after manually selected grid", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      // Two screenshares will happen in sequence
-      cold("-ft-ft", { f: false, t: true }),
-    );
-
-    // There was a screen-share that forced spotlight, then
-    // the user manually switch back to grid
-    schedule("---g", {
-      g: () => setGridMode("grid"),
-    });
-
-    // If we did want to respect manual selection, the expectation would be:
-    // const expectation = "ggsg";
-    const expectation = "ggsg-s";
-
-    expectObservable(gridMode$).toBe(expectation, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should switch back to grid mode when the remote screen share ends", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "f--t--f-";
-    const gridsMarble = "g--s--g-";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("can auto-switch to spotlight again after first screen share ends", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "ftft";
-    const gridsMarble = "gsgs";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("can switch manually to grid after screen share while manually in spotlight", () => {
-  withTestScheduler(({ cold, behavior, schedule, expectObservable }): void => {
-    // Initially, no one is sharing. Then the user manually switches to
-    // spotlight. After a screen share starts, the user manually switches to
-    // grid.
-    const shareMarbles = "  f-t-";
-    const setModeMarbles = "-s-g";
-    const expectation = "   gs-g";
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarbles, { f: false, t: true }),
-    );
-    schedule(setModeMarbles, {
       g: () => setGridMode("grid"),
       s: () => setGridMode("spotlight"),
     });
-
+    expectObservable(gridMode$).toBe(expectedGridMode, {
-    expectObservable(gridMode$).toBe(expectation, {
       g: "grid",
       s: "spotlight",
     });
   });
+}
+
+describe("default mode", () => {
+  test("uses grid layout by default", () =>
+    testLayoutSwitch({
+      expectedGridMode: "g",
+    }));
+
+  test("uses spotlight mode when window mode is flat", () =>
+    testLayoutSwitch({
+      windowMode: "      f",
+      expectedGridMode: "s",
+    }));
 });
 
-test("Should auto-switch to spotlight when in flat window mode", () => {
+test("allows switching modes manually", () =>
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
+  testLayoutSwitch({
-    const { gridMode$ } = createLayoutModeSwitch(
+    userSelection: "   --sgs",
-      scope,
+    expectedGridMode: "g-sgs",
-      behavior("naf", { n: "normal", a: "narrow", f: "flat" }),
+  }));
-      cold("f", { f: false, t: true }),
-    );
 
-    expectObservable(gridMode$).toBe("g-s-", {
+test("switches to spotlight mode when there is a remote screen share", () =>
-      g: "grid",
+  testLayoutSwitch({
-      s: "spotlight",
+    hasScreenShares: " n--y",
-    });
+    expectedGridMode: "g--s",
-  });
+  }));
-});
+
+test("can manually switch to grid when there is a screenshare", () =>
+  testLayoutSwitch({
+    hasScreenShares: " n-y",
+    userSelection: "   ---g",
+    expectedGridMode: "g-sg",
+  }));
+
+test("auto-switches after manually selecting grid", () =>
+  testLayoutSwitch({
+    // Two screenshares will happen in sequence. There is a screen share that
+    // forces spotlight, then the user manually switches back to grid.
+    hasScreenShares: " n-y-ny",
+    userSelection: "   ---g",
+    expectedGridMode: "g-sg-s",
+    // If we did want to respect manual selection, the expectation would be: g-sg
+  }));
+
+test("switches back to grid mode when the remote screen share ends", () =>
+  testLayoutSwitch({
+    hasScreenShares: " n--y--n",
+    expectedGridMode: "g--s--g",
+  }));
+
+test("auto-switches to spotlight again after first screen share ends", () =>
+  testLayoutSwitch({
+    hasScreenShares: " nyny",
+    expectedGridMode: "gsgs",
+  }));
+
+test("switches manually to grid after screen share while manually in spotlight", () =>
+  testLayoutSwitch({
+    // Initially, no one is sharing. Then the user manually switches to spotlight.
+    // After a screen share starts, the user manually switches to grid.
+    hasScreenShares: " n-y",
+    userSelection: "   -s-g",
+    expectedGridMode: "gs-g",
+  }));
+
+test("auto-switches to spotlight when in flat window mode", () =>
+  testLayoutSwitch({
+    // First normal, then narrow, then flat.
+    windowMode: "      nNf",
+    expectedGridMode: "g-s",
+  }));
+
+test("allows switching modes manually when in flat window mode", () =>
+  testLayoutSwitch({
+    // Window becomes flat, then user switches to grid and back.
+    // Finally the window returns to a normal shape.
+    windowMode: "      nf--n",
+    userSelection: "   --gs",
+    expectedGridMode: "gsgsg",
+  }));
+
+test("stays in spotlight while there are screen shares even when window mode changes", () =>
+  testLayoutSwitch({
+    windowMode: "      nfn",
+    hasScreenShares: " y",
+    expectedGridMode: "s",
+  }));
+
+test("ignores end of screen share until window mode returns to normal", () =>
+  testLayoutSwitch({
+    windowMode: "      nf-n",
+    hasScreenShares: " y-n",
+    expectedGridMode: "s--g",
+  }));

src/state/CallViewModel/LayoutSwitch.ts

@@ -6,121 +6,84 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  BehaviorSubject,
   combineLatest,
   map,
-  type Observable,
+  Subject,
-  scan,
+  startWith,
+  skipWhile,
+  switchMap,
 } from "rxjs";
-import { logger } from "matrix-js-sdk/lib/logger";
 
 import { type GridMode, type WindowMode } from "./CallViewModel.ts";
-import { type Behavior } from "../Behavior.ts";
+import { constant, type Behavior } from "../Behavior.ts";
 import { type ObservableScope } from "../ObservableScope.ts";
 
 /**
  * Creates a layout mode switch that allows switching between grid and spotlight modes.
- * The actual layout mode can be overridden to spotlight mode if there is a remote screen share active
+ * The actual layout mode might switch automatically to spotlight if there is a
- * or if the window mode is flat.
+ * remote screen share active or if the window mode is flat.
  *
  * @param scope - The observable scope to manage subscriptions.
- * @param windowMode$ - The current window mode observable.
+ * @param windowMode$ - The current window mode.
- * @param hasRemoteScreenShares$ - An observable indicating if there are remote screen shares active.
+ * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
  */
 export function createLayoutModeSwitch(
   scope: ObservableScope,
   windowMode$: Behavior<WindowMode>,
-  hasRemoteScreenShares$: Observable<boolean>,
+  hasRemoteScreenShares$: Behavior<boolean>,
 ): {
   gridMode$: Behavior<GridMode>;
   setGridMode: (value: GridMode) => void;
 } {
-  const gridModeUserSelection$ = new BehaviorSubject<GridMode>("grid");
+  const userSelection$ = new Subject<GridMode>();
-
   // Callback to set the grid mode desired by the user.
   // Notice that this is only a preference, the actual grid mode can be overridden
   // if there is a remote screen share active.
-  const setGridMode = (value: GridMode): void => {
+  const setGridMode = (value: GridMode): void => userSelection$.next(value);
-    gridModeUserSelection$.next(value);
+
-  };
+  /**
+   * The natural grid mode - the mode that the grid would prefer to be in,
+   * not accounting for the user's manual selections.
+   */
+  const naturalGridMode$ = scope.behavior<GridMode>(
+    combineLatest(
+      [hasRemoteScreenShares$, windowMode$],
+      (hasRemoteScreenShares, windowMode) =>
+        // When there are screen shares or the window is flat (as with a phone
+        // in landscape orientation), spotlight is a better experience.
+        // We want screen shares to be big and readable, and we want flipping
+        // your phone into landscape to be a quick way of maximising the
+        // spotlight tile.
+        hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid",
+    ),
+  );
+
   /**
    * The layout mode of the media tile grid.
    */
-  const gridMode$ =
+  const gridMode$ = scope.behavior<GridMode>(
-    // If the user hasn't selected spotlight and somebody starts screen sharing,
+    // Whenever the user makes a selection, we enter a new mode of behavior:
-    // automatically switch to spotlight mode and reset when screen sharing ends
+    userSelection$.pipe(
-    scope.behavior<GridMode>(
+      map((selection) => {
-      combineLatest([
+        if (selection === "grid")
-        gridModeUserSelection$,
+          // The user has selected grid mode. Start by respecting their choice,
-        hasRemoteScreenShares$,
+          // but then follow the natural mode again as soon as it matches.
-        windowMode$,
+          return naturalGridMode$.pipe(
-      ]).pipe(
+            skipWhile((naturalMode) => naturalMode !== selection),
-        // Scan to keep track if we have auto-switched already or not.
+            startWith(selection),
-        // To allow the user to override the auto-switch by selecting grid mode again.
+          );
-        scan<
-          [GridMode, boolean, WindowMode],
-          {
-            mode: GridMode;
-            /** Remember if the change was user driven or not */
-            hasAutoSwitched: boolean;
-            /** To know if it is new screen share or an already handled */
-            hasScreenShares: boolean;
-          }
-        >(
-          (prev, [userSelection, hasScreenShares, windowMode]) => {
-            const isFlatMode = windowMode === "flat";
 
-            // Always force spotlight in flat mode, grid layout is not supported
+        // The user has selected spotlight mode. If this matches the natural
-            // in that mode.
+        // mode, then follow the natural mode going forward.
-            // TODO: strange that we do that for flat mode but not for other modes?
+        return selection === naturalGridMode$.value
-            // TODO: Why is this not handled in layoutMedia$ like other window modes?
+          ? naturalGridMode$
-            if (isFlatMode) {
+          : constant(selection);
-              logger.debug(`Forcing spotlight mode, windowMode=${windowMode}`);
+      }),
-              return {
+      // Initially the mode of behavior is to just follow the natural grid mode.
-                mode: "spotlight",
+      startWith(naturalGridMode$),
-                hasAutoSwitched: prev.hasAutoSwitched,
+      // Switch between each mode of behavior.
-                hasScreenShares,
+      switchMap((mode$) => mode$),
-              };
-            }
-
-            // User explicitly chose spotlight.
-            // Respect that choice.
-            if (userSelection === "spotlight") {
-              return {
-                mode: "spotlight",
-                hasAutoSwitched: prev.hasAutoSwitched,
-                hasScreenShares,
-              };
-            }
-
-            // User has chosen grid mode. If a screen share starts, we will
-            // auto-switch to spotlight mode for better experience.
-            // But we only do it once, if the user switches back to grid mode,
-            // we respect that choice until they explicitly change it again.
-            const isNewShare = hasScreenShares && !prev.hasScreenShares;
-            if (isNewShare && !prev.hasAutoSwitched) {
-              return {
-                mode: "spotlight",
-                hasAutoSwitched: true,
-                hasScreenShares: true,
-              };
-            }
-
-            // Respect user's grid choice
-            // XXX If we want to forbid switching automatically again after we can
-            // return hasAutoSwitched: acc.hasAutoSwitched here instead of setting to false.
-            return {
-              mode: "grid",
-              hasAutoSwitched: false,
-              hasScreenShares,
-            };
-          },
-          // initial value
-          { mode: "grid", hasAutoSwitched: false, hasScreenShares: false },
     ),
-        map(({ mode }) => mode),
-      ),
-      "grid",
   );
 
   return {

src/state/CallViewModel/localMember/HomeserverConnected.test.ts

@@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => {
   // LLM generated test cases. They are a bit overkill but I improved the mocking so it is
   // easy enough to read them so I think they can stay.
   it("is false when sync state is not Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("remains false while membership status is not Connected even if sync is Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false); // membership still disconnected
+    expect(hsConnected.combined$.value).toBe(false); // membership still disconnected
   });
 
   it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // Make sync loop OK
     client.setSyncState(SyncState.Syncing);
     // Indicate probable leave before connection
     session.setProbablyLeft(true);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("becomes true only when all three conditions are satisfied", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // 1. Sync loop connected
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false); // not yet membership connected
+    expect(hsConnected.combined$.value).toBe(false); // not yet membership connected
     // 2. Membership connected
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true); // probablyLeft is false
+    expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false
   });
 
   it("drops back to false when sync loop leaves Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // Reach connected state
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Sync loop error => should flip false
     client.setSyncState(SyncState.Error);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("drops back to false when membership status becomes disconnected", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setMembershipStatus(Status.Disconnected);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("drops to false when ProbablyLeft is emitted after being true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Simulate clearing the flag (in realistic scenario membership manager would update)
     session.setProbablyLeft(false);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
   });
 
   it("composite sequence reflects each individual failure reason", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
 
     // Initially false (sync error + disconnected + not probably left)
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Fix sync only
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Fix membership
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Introduce probablyLeft -> false
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Restore notProbablyLeft -> true again
     session.setProbablyLeft(false);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Drop sync -> false
     client.setSyncState(SyncState.Error);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 });

src/state/CallViewModel/localMember/HomeserverConnected.ts

@@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test";
  */
 const logger = rootLogger.getChild("[HomeserverConnected]");
 
+export interface HomeserverConnected {
+  combined$: Behavior<boolean>;
+  rtsSession$: Behavior<Status>;
+}
+
 /**
  * Behavior representing whether we consider ourselves connected to the Matrix homeserver
  * for the purposes of a MatrixRTC session.
@@ -39,7 +44,7 @@ export function createHomeserverConnected$(
   client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
   matrixRTCSession: NodeStyleEventEmitter &
     Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
-): Behavior<boolean> {
+): HomeserverConnected {
   const syncing$ = (
     fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
   ).pipe(
@@ -47,12 +52,15 @@ export function createHomeserverConnected$(
     map(([state]) => state === SyncState.Syncing),
   );
 
-  const membershipConnected$ = fromEvent(
+  const rtsSession$ = scope.behavior<Status>(
-    matrixRTCSession,
+    fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
-    MembershipManagerEvent.StatusChanged,
+      map(() => matrixRTCSession.membershipStatus ?? Status.Unknown),
-  ).pipe(
+    ),
-    startWith(null),
+    Status.Unknown,
-    map(() => matrixRTCSession.membershipStatus === Status.Connected),
+  );
+
+  const membershipConnected$ = rtsSession$.pipe(
+    map((status) => status === Status.Connected),
   );
 
   // This is basically notProbablyLeft$
@@ -71,15 +79,13 @@ export function createHomeserverConnected$(
     map(() => matrixRTCSession.probablyLeft !== true),
   );
 
-  const connectedCombined$ = and$(
+  const combined$ = scope.behavior(
-    syncing$,
+    and$(syncing$, membershipConnected$, certainlyConnected$).pipe(
-    membershipConnected$,
-    certainlyConnected$,
-  ).pipe(
       tap((connected) => {
         logger.info(`Homeserver connected update: ${connected}`);
       }),
+    ),
   );
 
-  return scope.behavior(connectedCombined$);
+  return { combined$, rtsSession$ };
 }

src/state/CallViewModel/localMember/LocalMember.test.ts

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  Status as RTCMemberStatus,
   type LivekitTransport,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
@@ -14,11 +15,7 @@ import { describe, expect, it, vi } from "vitest";
 import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
 import { BehaviorSubject, map, of } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
-import {
+import { type LocalParticipant, type LocalTrack } from "livekit-client";
-  ConnectionState as LivekitConnectionState,
-  type LocalParticipant,
-  type LocalTrack,
-} from "livekit-client";
 
 import { MatrixRTCMode } from "../../../settings/settings";
 import {
@@ -29,15 +26,17 @@ import {
   withTestScheduler,
 } from "../../../utils/test";
 import {
+  TransportState,
   createLocalMembership$,
   enterRTCSession,
-  RTCBackendState,
+  PublishState,
-} from "./LocalMembership";
+  TrackState,
+} from "./LocalMember";
 import { MatrixRTCTransportMissingError } from "../../../utils/errors";
 import { Epoch, ObservableScope } from "../../ObservableScope";
 import { constant } from "../../Behavior";
 import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
-import { type Connection } from "../remoteMembers/Connection";
+import { ConnectionState, type Connection } from "../remoteMembers/Connection";
 import { type Publisher } from "./Publisher";
 
 const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
@@ -51,7 +50,7 @@ vi.mock("@livekit/components-core", () => ({
 
 describe("LocalMembership", () => {
   describe("enterRTCSession", () => {
-    it("It joins the correct Session", async () => {
+    it("It joins the correct Session", () => {
       const focusFromOlderMembership = {
         type: "livekit",
         livekit_service_url: "http://my-oldest-member-service-url.com",
@@ -107,7 +106,7 @@ describe("LocalMembership", () => {
         joinRoomSession: vi.fn(),
       }) as unknown as MatrixRTCSession;
 
-      await enterRTCSession(
+      enterRTCSession(
         mockedSession,
         {
           livekit_alias: "roomId",
@@ -136,7 +135,7 @@ describe("LocalMembership", () => {
       );
     });
 
-    it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
+    it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => {
       mockConfig({});
       vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
         "org.matrix.msc4143.rtc_foci": [
@@ -165,7 +164,7 @@ describe("LocalMembership", () => {
         joinRoomSession: vi.fn(),
       }) as unknown as MatrixRTCSession;
 
-      await enterRTCSession(
+      enterRTCSession(
         mockedSession,
         {
           livekit_alias: "roomId",
@@ -190,7 +189,6 @@ describe("LocalMembership", () => {
       leaveRoomSession: () => {},
     } as unknown as MatrixRTCSession,
     muteStates: mockMuteStates(),
-    isHomeserverConnected: constant(true),
     trackProcessorState$: constant({
       supported: false,
       processor: undefined,
@@ -198,20 +196,20 @@ describe("LocalMembership", () => {
     logger: logger,
     createPublisherFactory: vi.fn(),
     joinMatrixRTC: async (): Promise<void> => {},
-    homeserverConnected$: constant(true),
+    homeserverConnected: {
+      combined$: constant(true),
+      rtsSession$: constant(RTCMemberStatus.Connected),
+    },
   };
 
   it("throws error on missing RTC config error", () => {
     withTestScheduler(({ scope, hot, expectObservable }) => {
-      const goodTransport = {
+      const localTransport$ = scope.behavior<null | LivekitTransport>(
-        livekit_service_url: "other",
-      } as LivekitTransport;
-
-      const localTransport$ = scope.behavior<LivekitTransport>(
         hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
-        goodTransport,
+        null,
       );
 
+      // we do not need any connection data since we want to fail before reaching that.
       const mockConnectionManager = {
         transports$: scope.behavior(
           localTransport$.pipe(map((t) => new Epoch([t]))),
@@ -227,15 +225,11 @@ describe("LocalMembership", () => {
         connectionManager: mockConnectionManager,
         localTransport$,
       });
+      localMembership.requestJoinAndPublish();
 
-      expectObservable(localMembership.connectionState.livekit$).toBe("ne", {
+      expectObservable(localMembership.localMemberState$).toBe("ne", {
-        n: { state: RTCBackendState.WaitingForConnection },
+        n: TransportState.Waiting,
-        e: {
+        e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError),
-          state: RTCBackendState.Error,
-          error: expect.toSatisfy(
-            (e) => e instanceof MatrixRTCTransportMissingError,
-          ),
-        },
       });
     });
   });
@@ -247,33 +241,26 @@ describe("LocalMembership", () => {
     livekit_service_url: "b",
   } as LivekitTransport;
 
-  const connectionManagerData = new ConnectionManagerData();
+  const connectionTransportAConnected = {
-
-  connectionManagerData.add(
-    {
     livekitRoom: mockLivekitRoom({
       localParticipant: {
         isScreenShareEnabled: false,
         trackPublications: [],
       } as unknown as LocalParticipant,
     }),
-      state$: constant({
+    state$: constant(ConnectionState.LivekitConnected),
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: constant(LivekitConnectionState.Connected),
-      }),
     transport: aTransport,
-    } as unknown as Connection,
+  } as unknown as Connection;
-    [],
+  const connectionTransportAConnecting = {
-  );
+    ...connectionTransportAConnected,
-  connectionManagerData.add(
+    state$: constant(ConnectionState.LivekitConnecting),
-    {
+    livekitRoom: mockLivekitRoom({}),
-      state$: constant({
+  } as unknown as Connection;
-        state: "ConnectedToLkRoom",
+  const connectionTransportBConnected = {
-      }),
+    state$: constant(ConnectionState.LivekitConnected),
     transport: bTransport,
-    } as unknown as Connection,
+    livekitRoom: mockLivekitRoom({}),
-    [],
+  } as unknown as Connection;
-  );
 
   it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
     const scope = new ObservableScope();
@@ -281,13 +268,17 @@ describe("LocalMembership", () => {
     const localTransport$ = new BehaviorSubject(aTransport);
 
     const publishers: Publisher[] = [];
-
+    let seed = 0;
     defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
       () => {
+        const a = seed;
+        seed += 1;
+        logger.info(`creating [${a}]`);
         const p = {
-          stopPublishing: vi.fn(),
+          stopPublishing: vi.fn().mockImplementation(() => {
+            logger.info(`stopPublishing [${a}]`);
+          }),
           stopTracks: vi.fn(),
-          publishing$: constant(false),
         };
         publishers.push(p as unknown as Publisher);
         return p;
@@ -298,6 +289,9 @@ describe("LocalMembership", () => {
         typeof vi.fn
       >;
 
+    const connectionManagerData = new ConnectionManagerData();
+    connectionManagerData.add(connectionTransportAConnected, []);
+    connectionManagerData.add(connectionTransportBConnected, []);
     createLocalMembership$({
       scope,
       ...defaultCreateLocalMemberValues,
@@ -322,7 +316,7 @@ describe("LocalMembership", () => {
     await flushPromises();
     // stop all tracks after ending scopes
     expect(publishers[1].stopPublishing).toHaveBeenCalled();
-    expect(publishers[1].stopTracks).toHaveBeenCalled();
+    // expect(publishers[1].stopTracks).toHaveBeenCalled();
 
     defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
   });
@@ -357,6 +351,9 @@ describe("LocalMembership", () => {
         typeof vi.fn
       >;
 
+    const connectionManagerData = new ConnectionManagerData();
+    connectionManagerData.add(connectionTransportAConnected, []);
+    // connectionManagerData.add(connectionTransportB, []);
     const localMembership = createLocalMembership$({
       scope,
       ...defaultCreateLocalMemberValues,
@@ -367,15 +364,17 @@ describe("LocalMembership", () => {
     });
     await flushPromises();
     expect(publisherFactory).toHaveBeenCalledOnce();
-    expect(localMembership.tracks$.value.length).toBe(0);
+    // expect(localMembership.tracks$.value.length).toBe(0);
+    expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled();
     localMembership.startTracks();
     await flushPromises();
-    expect(localMembership.tracks$.value.length).toBe(2);
+    expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
+    // expect(localMembership.tracks$.value.length).toBe(2);
     scope.end();
     await flushPromises();
     // stop all tracks after ending scopes
     expect(publishers[0].stopPublishing).toHaveBeenCalled();
-    expect(publishers[0].stopTracks).toHaveBeenCalled();
+    // expect(publishers[0].stopTracks).toHaveBeenCalled();
     publisherFactory.mockClear();
   });
   // TODO add an integration test combining publisher and localMembership
@@ -383,10 +382,11 @@ describe("LocalMembership", () => {
   it("tracks livekit state correctly", async () => {
     const scope = new ObservableScope();
 
+    const connectionManagerData = new ConnectionManagerData();
     const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
-    const connectionManagerData$ = new BehaviorSubject<
+    const connectionManagerData$ = new BehaviorSubject(
-      Epoch<ConnectionManagerData>
+      new Epoch(connectionManagerData),
-    >(new Epoch(new ConnectionManagerData()));
+    );
     const publishers: Publisher[] = [];
 
     const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
@@ -432,61 +432,96 @@ describe("LocalMembership", () => {
     });
 
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(localMembership.localMemberState$.value).toStrictEqual(
-      state: RTCBackendState.WaitingForTransport,
+      TransportState.Waiting,
-    });
+    );
     localTransport$.next(aTransport);
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(localMembership.localMemberState$.value).toStrictEqual({
-      state: RTCBackendState.WaitingForConnection,
+      matrix: RTCMemberStatus.Connected,
+      media: { connection: null, tracks: TrackState.WaitingForUser },
     });
-    connectionManagerData$.next(new Epoch(connectionManagerData));
+
+    const connectionManagerData2 = new ConnectionManagerData();
+    connectionManagerData2.add(
+      // clone because we will mutate this later.
+      { ...connectionTransportAConnecting } as unknown as Connection,
+      [],
+    );
+
+    connectionManagerData$.next(new Epoch(connectionManagerData2));
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(localMembership.localMemberState$.value).toStrictEqual({
-      state: RTCBackendState.Initialized,
+      matrix: RTCMemberStatus.Connected,
+      media: {
+        connection: ConnectionState.LivekitConnecting,
+        tracks: TrackState.WaitingForUser,
+      },
     });
+
+    (
+      connectionManagerData2.getConnectionForTransport(aTransport)!
+        .state$ as BehaviorSubject<ConnectionState>
+    ).next(ConnectionState.LivekitConnected);
+    expect(localMembership.localMemberState$.value).toStrictEqual({
+      matrix: RTCMemberStatus.Connected,
+      media: {
+        connection: ConnectionState.LivekitConnected,
+        tracks: TrackState.WaitingForUser,
+      },
+    });
+
     expect(publisherFactory).toHaveBeenCalledOnce();
-    expect(localMembership.tracks$.value.length).toBe(0);
+    // expect(localMembership.tracks$.value.length).toBe(0);
 
     // -------
     localMembership.startTracks();
     // -------
 
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    // expect(localMembership.localMemberState$.value).toStrictEqual({
-      state: RTCBackendState.CreatingTracks,
+    //   matrix: RTCMemberStatus.Connected,
-    });
+    //   media: {
+    //     tracks: TrackState.Creating,
+    //     connection: ConnectionState.LivekitConnected,
+    //   },
+    // });
     createTrackResolver.resolve();
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(
-      state: RTCBackendState.ReadyToPublish,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    });
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.WaitingForUser);
 
     // -------
-    localMembership.requestConnect();
+    localMembership.requestJoinAndPublish();
     // -------
 
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(
-      state: RTCBackendState.WaitingToPublish,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    });
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
 
     publishResolver.resolve();
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(
-      state: RTCBackendState.Connected,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    });
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
+
     expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
 
-    expect(localMembership.connectionState.livekit$.isStopped).toBe(false);
+    expect(localMembership.localMemberState$.isStopped).toBe(false);
     scope.end();
     await flushPromises();
     // stays in connected state because it is stopped before the update to tracks update the state.
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
+    expect(
-      state: RTCBackendState.Connected,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    });
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
     // stop all tracks after ending scopes
     expect(publishers[0].stopPublishing).toHaveBeenCalled();
-    expect(publishers[0].stopTracks).toHaveBeenCalled();
+    // expect(publishers[0].stopTracks).toHaveBeenCalled();
   });
   // TODO add tests for matrix local matrix participation.
 });

src/state/CallViewModel/localMember/LocalMember.ts

@@ -6,15 +6,16 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  type LocalTrack,
   type Participant,
   ParticipantEvent,
   type LocalParticipant,
   type ScreenShareCaptureOptions,
-  ConnectionState,
+  RoomEvent,
+  MediaDeviceFailure,
 } from "livekit-client";
 import { observeParticipantEvents } from "@livekit/components-core";
 import {
+  Status as RTCSessionStatus,
   type LivekitTransport,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
@@ -24,10 +25,11 @@ import {
   combineLatest,
   distinctUntilChanged,
   from,
+  fromEvent,
   map,
   type Observable,
   of,
-  scan,
+  pairwise,
   startWith,
   switchMap,
   tap,
@@ -35,74 +37,73 @@ import {
 import { type Logger } from "matrix-js-sdk/lib/logger";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
 
-import { constant, type Behavior } from "../../Behavior";
+import { type Behavior } from "../../Behavior.ts";
-import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
+import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
-import { ObservableScope } from "../../ObservableScope";
+import { type ObservableScope } from "../../ObservableScope.ts";
-import { type Publisher } from "./Publisher";
+import { type Publisher } from "./Publisher.ts";
-import { type MuteStates } from "../../MuteStates";
+import { type MuteStates } from "../../MuteStates.ts";
-import { and$ } from "../../../utils/observable";
 import {
   ElementCallError,
+  FailToStartLivekitConnection,
   MembershipManagerError,
   UnknownCallError,
-} from "../../../utils/errors";
+} from "../../../utils/errors.ts";
-import { ElementWidgetActions, widget } from "../../../widget";
+import { ElementWidgetActions, widget } from "../../../widget.ts";
 import { getUrlParams } from "../../../UrlParams.ts";
 import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
 import { MatrixRTCMode } from "../../../settings/settings.ts";
 import { Config } from "../../../config/Config.ts";
-import { type Connection } from "../remoteMembers/Connection.ts";
+import {
+  ConnectionState,
+  type Connection,
+  type FailedToStartError,
+} from "../remoteMembers/Connection.ts";
+import { type HomeserverConnected } from "./HomeserverConnected.ts";
+import { and$ } from "../../../utils/observable.ts";
 
-export enum RTCBackendState {
+export enum TransportState {
-  Error = "error",
   /** Not even a transport is available to the LocalMembership */
-  WaitingForTransport = "waiting_for_transport",
+  Waiting = "transport_waiting",
-  /** A connection appeared so we can initialise the publisher */
-  WaitingForConnection = "waiting_for_connection",
-  /** Connection and transport arrived, publisher Initialized */
-  Initialized = "Initialized",
-  CreatingTracks = "creating_tracks",
-  ReadyToPublish = "ready_to_publish",
-  WaitingToPublish = "waiting_to_publish",
-  Connected = "connected",
-  Disconnected = "disconnected",
-  Disconnecting = "disconnecting",
 }
 
-type LocalMemberRtcBackendState =
+export enum PublishState {
-  | { state: RTCBackendState.Error; error: ElementCallError }
+  WaitingForUser = "publish_waiting_for_user",
-  | { state: RTCBackendState.WaitingForTransport }
+  // XXX: This state is removed for now since we do not have full control over
-  | { state: RTCBackendState.WaitingForConnection }
+  // track publication anymore with the publisher abstraction, might come back in the future?
-  | { state: RTCBackendState.Initialized }
+  // /** Implies lk connection is connected */
-  | { state: RTCBackendState.CreatingTracks }
+  // Starting = "publish_start_publishing",
-  | { state: RTCBackendState.ReadyToPublish }
+  /** Implies lk connection is connected */
-  | { state: RTCBackendState.WaitingToPublish }
+  Publishing = "publish_publishing",
-  | { state: RTCBackendState.Connected }
-  | { state: RTCBackendState.Disconnected }
-  | { state: RTCBackendState.Disconnecting };
-
-export enum MatrixState {
-  WaitingForTransport = "waiting_for_transport",
-  Ready = "ready",
-  Connecting = "connecting",
-  Connected = "connected",
-  Disconnected = "disconnected",
-  Error = "Error",
 }
 
-type LocalMemberMatrixState =
+// TODO not sure how to map that correctly with the
-  | { state: MatrixState.Connected }
+// new publisher that does not manage tracks itself anymore
-  | { state: MatrixState.WaitingForTransport }
+export enum TrackState {
-  | { state: MatrixState.Ready }
+  /** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */
-  | { state: MatrixState.Connecting }
+  WaitingForUser = "tracks_waiting_for_user",
-  | { state: MatrixState.Disconnected }
+  // XXX: This state is removed for now since we do not have full control over
-  | { state: MatrixState.Error; error: Error };
+  // track creation anymore with the publisher abstraction, might come back in the future?
-
+  // /** Implies lk connection is connected */
-export interface LocalMemberConnectionState {
+  // Creating = "tracks_creating",
-  livekit$: Behavior<LocalMemberRtcBackendState>;
+  /** Implies lk connection is connected */
-  matrix$: Behavior<LocalMemberMatrixState>;
+  Ready = "tracks_ready",
 }
 
+export type LocalMemberMediaState =
+  | {
+      tracks: TrackState;
+      connection: ConnectionState | FailedToStartError;
+    }
+  | PublishState
+  | ElementCallError;
+export type LocalMemberState =
+  | ElementCallError
+  | TransportState.Waiting
+  | {
+      media: LocalMemberMediaState;
+      matrix: ElementCallError | RTCSessionStatus;
+    };
+
 /*
  * - get well known
  * - get oldest membership
@@ -122,8 +123,8 @@ interface Props {
   muteStates: MuteStates;
   connectionManager: IConnectionManager;
   createPublisherFactory: (connection: Connection) => Publisher;
-  joinMatrixRTC: (transport: LivekitTransport) => Promise<void>;
+  joinMatrixRTC: (transport: LivekitTransport) => void;
-  homeserverConnected$: Behavior<boolean>;
+  homeserverConnected: HomeserverConnected;
   localTransport$: Behavior<LivekitTransport | null>;
   matrixRTCSession: Pick<
     MatrixRTCSession,
@@ -149,7 +150,7 @@ export const createLocalMembership$ = ({
   scope,
   connectionManager,
   localTransport$: localTransportCanThrow$,
-  homeserverConnected$,
+  homeserverConnected,
   createPublisherFactory,
   joinMatrixRTC,
   logger: parentLogger,
@@ -157,32 +158,33 @@ export const createLocalMembership$ = ({
   matrixRTCSession,
 }: Props): {
   /**
-   * This starts audio and video tracks. They will be reused when calling `requestConnect`.
+   * This request to start audio and video tracks.
+   * Can be called early to pre-emptively get media permissions and start devices.
    */
-  startTracks: () => Behavior<LocalTrack[]>;
+  startTracks: () => void;
   /**
-   * This sets a inner state (shouldConnect) to true and instructs the js-sdk and livekit to keep the user
+   * This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user
    * connected to matrix and livekit.
    */
-  requestConnect: () => void;
+  requestJoinAndPublish: () => void;
   requestDisconnect: () => void;
-  connectionState: LocalMemberConnectionState;
+  localMemberState$: Behavior<LocalMemberState>;
   sharingScreen$: Behavior<boolean>;
   /**
    * Callback to toggle screen sharing. If null, screen sharing is not possible.
    */
   toggleScreenSharing: (() => void) | null;
-  tracks$: Behavior<LocalTrack[]>;
+  // tracks$: Behavior<LocalTrack[]>;
   participant$: Behavior<LocalParticipant | null>;
   connection$: Behavior<Connection | null>;
-  connected$: Behavior<boolean>;
+  /** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
-  // this needs to be discussed
+   * Direct translation to the js-sdk membership manager connection `Status`.
-  /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
-   * @deprecated use state instead
    */
   reconnecting$: Behavior<boolean>;
+  /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
+   * Direct translation to the js-sdk membership manager connection `Status`.
+   */
+  disconnected$: Behavior<boolean>;
 } => {
   const logger = parentLogger.getChild("[LocalMembership]");
   logger.debug(`Creating local membership..`);
@@ -201,7 +203,7 @@ export const createLocalMembership$ = ({
               : new Error("Unknown error from localTransport"),
           );
         }
-        setLivekitError(error);
+        setTransportError(error);
         return of(null);
       }),
     ),
@@ -228,94 +230,59 @@ export const createLocalMembership$ = ({
     ),
   );
 
-  const localConnectionState$ = localConnection$.pipe(
+  // Tracks error that happen when creating the local tracks.
-    switchMap((connection) => (connection ? connection.state$ : of(null))),
+  const mediaErrors$ = localConnection$.pipe(
-  );
+    switchMap((connection) => {
-
+      if (!connection) {
-  // /**
+        return of(null);
-  //  * Whether we are "fully" connected to the call. Accounts for both the
+      } else {
-  //  * connection to the MatrixRTC session and the LiveKit publish connection.
+        return fromEvent(
-  //  */
+          connection.livekitRoom,
-  const connected$ = scope.behavior(
+          RoomEvent.MediaDevicesError,
-    and$(
+          (error: Error) => {
-      homeserverConnected$.pipe(
+            return MediaDeviceFailure.getFailure(error) ?? null;
-        tap((v) => logger.debug("matrix: Connected state changed", v)),
+          },
-      ),
-      localConnectionState$.pipe(
-        switchMap((state) => {
-          logger.debug("livekit: Connected state changed", state);
-          if (!state) return of(false);
-          if (state.state === "ConnectedToLkRoom") {
-            logger.debug(
-              "livekit: Connected state changed (inner livekitConnectionState$)",
-              state.livekitConnectionState$.value,
-            );
-            return state.livekitConnectionState$.pipe(
-              map((lkState) => lkState === ConnectionState.Connected),
         );
       }
-          return of(false);
     }),
-      ),
-    ).pipe(tap((v) => logger.debug("combined: Connected state changed", v))),
   );
 
+  mediaErrors$.pipe(scope.bind()).subscribe((error) => {
+    if (error) {
+      logger.error(`Failed to create local tracks:`, error);
+      setMatrixError(
+        // TODO is it fatal? Do we need to create a new Specialized Error?
+        new UnknownCallError(new Error(`Media device error: ${error}`)),
+      );
+    }
+  });
   // MATRIX RELATED
 
-  // /**
-  //  * Whether we should tell the user that we're reconnecting to the call.
-  //  */
-  // DISCUSSION is there a better way to do this?
-  // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
-  const reconnecting$ = scope.behavior(
-    connected$.pipe(
-      // We are reconnecting if we previously had some successful initial
-      // connection but are now disconnected
-      scan(
-        ({ connectedPreviously }, connectedNow) => ({
-          connectedPreviously: connectedPreviously || connectedNow,
-          reconnecting: connectedPreviously && !connectedNow,
-        }),
-        { connectedPreviously: false, reconnecting: false },
-      ),
-      map(({ reconnecting }) => reconnecting),
-    ),
-  );
-
   // This should be used in a combineLatest with publisher$ to connect.
   // to make it possible to call startTracks before the preferredTransport$ has resolved.
   const trackStartRequested = Promise.withResolvers<void>();
 
   // This should be used in a combineLatest with publisher$ to connect.
   // to make it possible to call startTracks before the preferredTransport$ has resolved.
-  const connectRequested$ = new BehaviorSubject(false);
+  const joinAndPublishRequested$ = new BehaviorSubject(false);
 
   /**
    * The publisher is stored in here an abstracts creating and publishing tracks.
    */
   const publisher$ = new BehaviorSubject<Publisher | null>(null);
-  /**
-   * Extract the tracks from the published. Also reacts to changing publishers.
-   */
-  const tracks$ = scope.behavior(
-    publisher$.pipe(switchMap((p) => (p?.tracks$ ? p.tracks$ : constant([])))),
-  );
-  const publishing$ = scope.behavior(
-    publisher$.pipe(switchMap((p) => p?.publishing$ ?? constant(false))),
-  );
 
-  const startTracks = (): Behavior<LocalTrack[]> => {
+  const startTracks = (): void => {
     trackStartRequested.resolve();
-    return tracks$;
+    // This used to return the tracks, but now they are only accessible via the publisher.
   };
 
-  const requestConnect = (): void => {
+  const requestJoinAndPublish = (): void => {
     trackStartRequested.resolve();
-    connectRequested$.next(true);
+    joinAndPublishRequested$.next(true);
   };
 
   const requestDisconnect = (): void => {
-    connectRequested$.next(false);
+    joinAndPublishRequested$.next(false);
   };
 
   // Take care of the publisher$
@@ -332,7 +299,7 @@ export const createLocalMembership$ = ({
       // Clean-up callback
       return Promise.resolve(async (): Promise<void> => {
         await publisher.stopPublishing();
-        publisher.stopTracks();
+        await publisher.stopTracks();
       });
     }
   });
@@ -341,13 +308,16 @@ export const createLocalMembership$ = ({
   // `tracks$` will update once they are ready.
   scope.reconcile(
     scope.behavior(
-      combineLatest([publisher$, tracks$, from(trackStartRequested.promise)]),
+      combineLatest([
+        publisher$ /*, tracks$*/,
+        from(trackStartRequested.promise),
+      ]),
       null,
     ),
     async (valueIfReady) => {
       if (!valueIfReady) return;
-      const [publisher, tracks] = valueIfReady;
+      const [publisher] = valueIfReady;
-      if (publisher && tracks.length === 0) {
+      if (publisher) {
         await publisher.createAndSetupTracks().catch((e) => logger.error(e));
       }
     },
@@ -355,140 +325,215 @@ export const createLocalMembership$ = ({
 
   // Based on `connectRequested$` we start publishing tracks. (once they are there!)
   scope.reconcile(
-    scope.behavior(combineLatest([publisher$, tracks$, connectRequested$])),
+    scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])),
-    async ([publisher, tracks, shouldConnect]) => {
+    async ([publisher, shouldJoinAndPublish]) => {
-      if (shouldConnect === publisher?.publishing$.value) return;
+      // Get the current publishing state to avoid redundant calls.
-      if (tracks.length !== 0 && shouldConnect) {
+      const isPublishing = publisher?.shouldPublish === true;
+      if (shouldJoinAndPublish && !isPublishing) {
         try {
           await publisher?.startPublishing();
         } catch (error) {
-          setLivekitError(error as ElementCallError);
+          const message =
+            error instanceof Error ? error.message : String(error);
+          setPublishError(new FailToStartLivekitConnection(message));
         }
-      } else if (tracks.length !== 0 && !shouldConnect) {
+      } else if (isPublishing) {
         try {
           await publisher?.stopPublishing();
         } catch (error) {
-          setLivekitError(new UnknownCallError(error as Error));
+          setPublishError(new UnknownCallError(error as Error));
         }
       }
     },
   );
 
-  const fatalLivekitError$ = new BehaviorSubject<ElementCallError | null>(null);
+  // STATE COMPUTATION
-  const setLivekitError = (e: ElementCallError): void => {
+
-    if (fatalLivekitError$.value !== null)
+  // These are non fatal since we can join a room and concume media even though publishing failed.
-      logger.error("Multiple Livkit Errors:", e);
+  const publishError$ = new BehaviorSubject<ElementCallError | null>(null);
-    else fatalLivekitError$.next(e);
+  const setPublishError = (e: ElementCallError): void => {
+    if (publishError$.value !== null) {
+      logger.error("Multiple Media Errors:", e);
+    } else {
+      publishError$.next(e);
+    }
   };
-  const livekitState$: Behavior<LocalMemberRtcBackendState> = scope.behavior(
+
+  const fatalTransportError$ = new BehaviorSubject<ElementCallError | null>(
+    null,
+  );
+
+  const setTransportError = (e: ElementCallError): void => {
+    if (fatalTransportError$.value !== null) {
+      logger.error("Multiple Transport Errors:", e);
+    } else {
+      fatalTransportError$.next(e);
+    }
+  };
+
+  const localConnectionState$ = localConnection$.pipe(
+    switchMap((connection) => (connection ? connection.state$ : of(null))),
+  );
+
+  const mediaState$: Behavior<LocalMemberMediaState> = scope.behavior(
     combineLatest([
-      publisher$,
+      localConnectionState$,
       localTransport$,
-      tracks$.pipe(
+      joinAndPublishRequested$,
-        tap((t) => {
-          logger.info("tracks$: ", t);
-        }),
-      ),
-      publishing$,
-      connectRequested$,
       from(trackStartRequested.promise).pipe(
         map(() => true),
         startWith(false),
       ),
-      fatalLivekitError$,
     ]).pipe(
       map(
         ([
-          publisher,
+          localConnectionState,
           localTransport,
-          tracks,
+          shouldPublish,
-          publishing,
-          shouldConnect,
           shouldStartTracks,
-          error,
         ]) => {
-          // read this:
+          if (!localTransport) return null;
-          // if(!<A>) return {state: ...}
+          const trackState: TrackState = shouldStartTracks
-          // if(!<B>) return {state: <MyState>}
+            ? TrackState.Ready
-          //
+            : TrackState.WaitingForUser;
-          // as:
+
-          // We do have <A> but not yet <B> so we are in <MyState>
+          if (
-          if (error !== null) return { state: RTCBackendState.Error, error };
+            localConnectionState !== ConnectionState.LivekitConnected ||
-          const hasTracks = tracks.length > 0;
+            trackState !== TrackState.Ready
-          if (!localTransport)
+          )
-            return { state: RTCBackendState.WaitingForTransport };
+            return {
-          if (!publisher)
+              connection: localConnectionState,
-            return { state: RTCBackendState.WaitingForConnection };
+              tracks: trackState,
-          if (!shouldStartTracks) return { state: RTCBackendState.Initialized };
+            };
-          if (!hasTracks) return { state: RTCBackendState.CreatingTracks };
+          if (!shouldPublish) return PublishState.WaitingForUser;
-          if (!shouldConnect) return { state: RTCBackendState.ReadyToPublish };
+          // if (!publishing) return PublishState.Starting;
-          if (!publishing) return { state: RTCBackendState.WaitingToPublish };
+          return PublishState.Publishing;
-          return { state: RTCBackendState.Connected };
         },
       ),
       distinctUntilChanged(deepCompare),
     ),
   );
-
   const fatalMatrixError$ = new BehaviorSubject<ElementCallError | null>(null);
   const setMatrixError = (e: ElementCallError): void => {
-    if (fatalMatrixError$.value !== null)
+    if (fatalMatrixError$.value !== null) {
       logger.error("Multiple Matrix Errors:", e);
-    else fatalMatrixError$.next(e);
+    } else {
+      fatalMatrixError$.next(e);
+    }
   };
-  const matrixState$: Behavior<LocalMemberMatrixState> = scope.behavior(
+
+  const localMemberState$ = scope.behavior<LocalMemberState>(
     combineLatest([
-      localTransport$,
+      mediaState$,
-      connectRequested$,
+      homeserverConnected.rtsSession$,
-      homeserverConnected$,
+      fatalMatrixError$,
+      fatalTransportError$,
+      publishError$,
     ]).pipe(
-      map(([localTransport, connectRequested, homeserverConnected]) => {
+      map(
-        if (!localTransport) return { state: MatrixState.WaitingForTransport };
+        ([
-        if (!connectRequested) return { state: MatrixState.Ready };
+          mediaState,
-        if (!homeserverConnected) return { state: MatrixState.Connecting };
+          rtcSessionStatus,
-        return { state: MatrixState.Connected };
+          fatalMatrixError,
-      }),
+          fatalTransportError,
+          publishError,
+        ]) => {
+          if (fatalTransportError !== null) return fatalTransportError;
+          // `mediaState` will be 'null' until the transport/connection appears.
+          if (mediaState && rtcSessionStatus)
+            return {
+              matrix: fatalMatrixError ?? rtcSessionStatus,
+              media: publishError ?? mediaState,
+            };
+          return TransportState.Waiting;
+        },
+      ),
     ),
   );
 
-  // Keep matrix rtc session in sync with localTransport$, connectRequested$ and muteStates.video.enabled$
+  /**
+   * Whether we are "fully" connected to the call. Accounts for both the
+   * connection to the MatrixRTC session and the LiveKit publish connection.
+   */
+  const matrixAndLivekitConnected$ = scope.behavior(
+    and$(
+      homeserverConnected.combined$,
+      localConnectionState$.pipe(
+        map((state) => state === ConnectionState.LivekitConnected),
+      ),
+    ).pipe(
+      tap((v) => logger.debug("livekit+matrix: Connected state changed", v)),
+    ),
+  );
+
+  /**
+   * Whether we should tell the user that we're reconnecting to the call.
+   */
+  const reconnecting$ = scope.behavior(
+    matrixAndLivekitConnected$.pipe(
+      pairwise(),
+      map(([prev, current]) => prev === true && current === false),
+    ),
+    false,
+  );
+
+  // inform the widget about the connect and disconnect intent from the user.
+  scope
+    .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [
+      undefined,
+      joinAndPublishRequested$.value,
+    ])
+    .subscribe(([prev, current]) => {
+      if (!widget) return;
+      // JOIN prev=false (was left) => current-true (now joiend)
+      if (!prev && current) {
+        widget.api.transport
+          .send(ElementWidgetActions.JoinCall, {})
+          .catch((e) => {
+            logger.error("Failed to send join action", e);
+          });
+      }
+      // LEAVE prev=false (was joined) => current-true (now left)
+      if (prev && !current) {
+        widget.api.transport
+          .send(ElementWidgetActions.HangupCall, {})
+          .catch((e) => {
+            logger.error("Failed to send hangup action", e);
+          });
+      }
+    });
+
+  combineLatest([muteStates.video.enabled$, homeserverConnected.combined$])
+    .pipe(scope.bind())
+    .subscribe(([videoEnabled, connected]) => {
+      if (!connected) return;
+      void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio");
+    });
+
+  // Keep matrix rtc session in sync with localTransport$, connectRequested$
   scope.reconcile(
-    scope.behavior(combineLatest([localTransport$, connectRequested$])),
+    scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])),
     async ([transport, shouldConnect]) => {
+      if (!transport) return;
+      // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
       if (!shouldConnect) return;
 
-      if (!transport) return;
       try {
-        await joinMatrixRTC(transport);
+        joinMatrixRTC(transport);
       } catch (error) {
         logger.error("Error entering RTC session", error);
         if (error instanceof Error)
           setMatrixError(new MembershipManagerError(error));
       }
 
-      // Update our member event when our mute state changes.
+      return Promise.resolve(async (): Promise<void> => {
-      const callIntentScope = new ObservableScope();
-      // because this uses its own scope, we can start another reconciliation for the duration of one connection.
-      callIntentScope.reconcile(
-        muteStates.video.enabled$,
-        async (videoEnabled) =>
-          matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
-      );
-
-      return async (): Promise<void> => {
-        callIntentScope.end();
         try {
-          // Update matrixRTCSession to allow udpating the transport without leaving the session!
+          // TODO Update matrixRTCSession to allow udpating the transport without leaving the session!
-          await matrixRTCSession.leaveRoomSession();
+          await matrixRTCSession.leaveRoomSession(1000);
         } catch (e) {
           logger.error("Error leaving RTC session", e);
         }
-        try {
+      });
-          await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
-        } catch (e) {
-          logger.error("Failed to send hangup action", e);
-        }
-      };
     },
   );
 
@@ -503,7 +548,7 @@ export const createLocalMembership$ = ({
   // pause tracks during the initial joining sequence too until we're sure
   // that our own media is displayed on screen.
   // TODO refactor this based no livekitState$
-  combineLatest([participant$, homeserverConnected$])
+  combineLatest([participant$, homeserverConnected.combined$])
     .pipe(scope.bind())
     .subscribe(([participant, connected]) => {
       if (!participant) return;
@@ -588,16 +633,20 @@ export const createLocalMembership$ = ({
 
   return {
     startTracks,
-    requestConnect,
+    requestJoinAndPublish,
     requestDisconnect,
-    connectionState: {
+    localMemberState$,
-      livekit$: livekitState$,
-      matrix$: matrixState$,
-    },
-    tracks$,
     participant$,
+<<<<<<< HEAD:src/state/CallViewModel/localMember/LocalMembership.ts
     connected$,
+=======
+>>>>>>> livekit:src/state/CallViewModel/localMember/LocalMember.ts
     reconnecting$,
+    disconnected$: scope.behavior(
+      homeserverConnected.rtsSession$.pipe(
+        map((state) => state === RTCSessionStatus.Disconnected),
+      ),
+    ),
     sharingScreen$,
     toggleScreenSharing,
     connection$: localConnection$,
@@ -632,11 +681,11 @@ interface EnterRTCSessionOptions {
  * @throws If the widget could not send ElementWidgetActions.JoinCall action.
  */
 // Exported for unit testing
-export async function enterRTCSession(
+export function enterRTCSession(
   rtcSession: MatrixRTCSession,
   transport: LivekitTransport,
   { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
-): Promise<void> {
+): void {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -675,7 +724,4 @@ export async function enterRTCSession(
       unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
     },
   );
-  if (widget) {
-    await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
-  }
 }

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -85,7 +85,7 @@ export const createLocalTransport$ = ({
    * The transport that we would personally prefer to publish on (if not for the
    * transport preferences of others, perhaps).
    *
-   * @throws
+   * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
    */
   const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
     customLivekitUrl.value$.pipe(

src/state/CallViewModel/localMember/Publisher.test.ts

@@ -5,66 +5,320 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
+import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
 import {
-  afterEach,
+  ConnectionState as LivekitConnectionState,
-  beforeEach,
+  LocalParticipant,
-  describe,
+  type LocalTrack,
-  expect,
+  type LocalTrackPublication,
-  it,
+  ParticipantEvent,
-  type Mock,
+  Track,
-  vi,
+} from "livekit-client";
-} from "vitest";
+import { BehaviorSubject } from "rxjs";
-import { ConnectionState as LivekitConenctionState } from "livekit-client";
-import { type BehaviorSubject } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { ObservableScope } from "../../ObservableScope";
 import { constant } from "../../Behavior";
 import {
+  flushPromises,
   mockLivekitRoom,
-  mockLocalParticipant,
   mockMediaDevices,
 } from "../../../utils/test";
 import { Publisher } from "./Publisher";
-import {
+import { type Connection } from "../remoteMembers/Connection";
-  type Connection,
-  type ConnectionState,
-} from "../remoteMembers/Connection";
 import { type MuteStates } from "../../MuteStates";
-import { FailToStartLivekitConnection } from "../../../utils/errors";
 
-describe("Publisher", () => {
 let scope: ObservableScope;
-  let connection: Connection;
+
-  let muteStates: MuteStates;
 beforeEach(() => {
-    muteStates = {
-      audio: {
-        enabled$: constant(false),
-        unsetHandler: vi.fn(),
-        setHandler: vi.fn(),
-      },
-      video: {
-        enabled$: constant(false),
-        unsetHandler: vi.fn(),
-        setHandler: vi.fn(),
-      },
-    } as unknown as MuteStates;
   scope = new ObservableScope();
-    connection = {
-      state$: constant({
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: constant(LivekitConenctionState.Connected),
-      }),
-      livekitRoom: mockLivekitRoom({
-        localParticipant: mockLocalParticipant({}),
-      }),
-    } as unknown as Connection;
 });
 
 afterEach(() => scope.end());
 
-  it("throws if livekit room could not publish", async () => {
+function createMockLocalTrack(source: Track.Source): LocalTrack {
+  const track = {
+    source,
+    isMuted: false,
+    isUpstreamPaused: false,
+  } as Partial<LocalTrack> as LocalTrack;
+
+  vi.mocked(track).mute = vi.fn().mockImplementation(() => {
+    track.isMuted = true;
+  });
+  vi.mocked(track).unmute = vi.fn().mockImplementation(() => {
+    track.isMuted = false;
+  });
+  vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => {
+    // @ts-expect-error - for that test we want to set isUpstreamPaused directly
+    track.isUpstreamPaused = true;
+  });
+  vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => {
+    // @ts-expect-error - for that test we want to set isUpstreamPaused directly
+    track.isUpstreamPaused = false;
+  });
+
+  return track;
+}
+
+function createMockMuteState(enabled$: BehaviorSubject<boolean>): {
+  enabled$: BehaviorSubject<boolean>;
+  setHandler: (h: (enabled: boolean) => void) => void;
+  unsetHandler: () => void;
+} {
+  let currentHandler = (enabled: boolean): void => {};
+
+  const ms = {
+    enabled$,
+    setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => {
+      currentHandler = h;
+    }),
+    unsetHandler: vi.fn().mockImplementation(() => {
+      currentHandler = (enabled: boolean): void => {};
+    }),
+  };
+  // forward enabled$ emissions to the current handler
+  enabled$.subscribe((enabled) => {
+    logger.info(`MockMuteState: enabled changed to ${enabled}`);
+    currentHandler(enabled);
+  });
+
+  return ms;
+}
+
+let connection: Connection;
+let muteStates: MuteStates;
+let localParticipant: LocalParticipant;
+let audioEnabled$: BehaviorSubject<boolean>;
+let videoEnabled$: BehaviorSubject<boolean>;
+let trackPublications: LocalTrackPublication[];
+// use it to control when track creation resolves, default to resolved
+let createTrackLock: Promise<void>;
+
+beforeEach(() => {
+  trackPublications = [];
+  audioEnabled$ = new BehaviorSubject(false);
+  videoEnabled$ = new BehaviorSubject(false);
+  createTrackLock = Promise.resolve();
+
+  muteStates = {
+    audio: createMockMuteState(audioEnabled$),
+    video: createMockMuteState(videoEnabled$),
+  } as unknown as MuteStates;
+
+  const mockSendDataPacket = vi.fn();
+  const mockEngine = {
+    client: {
+      sendUpdateLocalMetadata: vi.fn(),
+    },
+    on: vi.fn().mockReturnThis(),
+    sendDataPacket: mockSendDataPacket,
+  };
+
+  localParticipant = new LocalParticipant(
+    "local-sid",
+    "local-identity",
+    // @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic
+    mockEngine,
+    {
+      adaptiveStream: true,
+      dynacase: false,
+      audioCaptureDefaults: {},
+      videoCaptureDefaults: {},
+      stopLocalTrackOnUnpublish: true,
+      reconnectPolicy: "always",
+      disconnectOnPageLeave: true,
+    },
+    new Map(),
+    {},
+  );
+
+  vi.mocked(localParticipant).createTracks = vi
+    .fn()
+    .mockImplementation(async (opts) => {
+      const tracks: LocalTrack[] = [];
+      if (opts.audio) {
+        tracks.push(createMockLocalTrack(Track.Source.Microphone));
+      }
+      if (opts.video) {
+        tracks.push(createMockLocalTrack(Track.Source.Camera));
+      }
+      await createTrackLock;
+      return tracks;
+    });
+
+  vi.mocked(localParticipant).publishTrack = vi
+    .fn()
+    .mockImplementation(async (track: LocalTrack) => {
+      const pub = {
+        track,
+        source: track.source,
+        mute: track.mute,
+        unmute: track.unmute,
+      } as Partial<LocalTrackPublication> as LocalTrackPublication;
+      trackPublications.push(pub);
+      localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub);
+      return Promise.resolve(pub);
+    });
+
+  vi.mocked(localParticipant).getTrackPublication = vi
+    .fn()
+    .mockImplementation((source: Track.Source) => {
+      return trackPublications.find((pub) => pub.track?.source === source);
+    });
+
+  connection = {
+    state$: constant({
+      state: "ConnectedToLkRoom",
+      livekitConnectionState$: constant(LivekitConnectionState.Connected),
+    }),
+    livekitRoom: mockLivekitRoom({
+      localParticipant: localParticipant,
+    }),
+  } as unknown as Connection;
+});
+
+describe("Publisher", () => {
+  let publisher: Publisher;
+
+  beforeEach(() => {
+    publisher = new Publisher(
+      scope,
+      connection,
+      mockMediaDevices({}),
+      muteStates,
+      constant({ supported: false, processor: undefined }),
+      logger,
+    );
+  });
+
+  afterEach(() => {});
+
+  it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
+    const createTracksSpy = vi.spyOn(
+      connection.livekitRoom.localParticipant,
+      "createTracks",
+    );
+
+    audioEnabled$.next(false);
+    videoEnabled$.next(false);
+    await publisher.createAndSetupTracks();
+
+    expect(createTracksSpy).not.toHaveBeenCalled();
+  });
+
+  it("Should minimize permission request by querying create at once", async () => {
+    const enableCameraAndMicrophoneSpy = vi.spyOn(
+      localParticipant,
+      "enableCameraAndMicrophone",
+    );
+    const createTracksSpy = vi.spyOn(localParticipant, "createTracks");
+
+    audioEnabled$.next(true);
+    videoEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+    await flushPromises();
+
+    expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled();
+
+    // It should create both at once
+    expect(createTracksSpy).toHaveBeenCalledWith({
+      audio: true,
+      video: true,
+    });
+  });
+
+  it("Ensure no data is streamed until publish has been called", async () => {
+    audioEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+
+    // The track should be created and paused
+    expect(localParticipant.createTracks).toHaveBeenCalledWith({
+      audio: true,
+      video: undefined,
+    });
+    await flushPromises();
+    expect(localParticipant.publishTrack).toHaveBeenCalled();
+
+    await flushPromises();
+    const track = localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track;
+    expect(track).toBeDefined();
+    expect(track!.pauseUpstream).toHaveBeenCalled();
+    expect(track!.isUpstreamPaused).toBe(true);
+  });
+
+  it("Ensure resume upstream when published is called", async () => {
+    videoEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+    // await flushPromises();
+    await publisher.startPublishing();
+
+    const track = localParticipant.getTrackPublication(
+      Track.Source.Camera,
+    )?.track;
+    expect(track).toBeDefined();
+    // expect(track.pauseUpstream).toHaveBeenCalled();
+    expect(track!.isUpstreamPaused).toBe(false);
+  });
+
+  describe("Mute states", () => {
+    let publisher: Publisher;
+    beforeEach(() => {
+      publisher = new Publisher(
+        scope,
+        connection,
+        mockMediaDevices({}),
+        muteStates,
+        constant({ supported: false, processor: undefined }),
+        logger,
+      );
+    });
+
+    test.each([
+      { mutes: { audioEnabled: true, videoEnabled: false } },
+      { mutes: { audioEnabled: true, videoEnabled: false } },
+    ])("only create the tracks that are unmuted $mutes", async ({ mutes }) => {
+      // Ensure all muted
+      audioEnabled$.next(mutes.audioEnabled);
+      videoEnabled$.next(mutes.videoEnabled);
+
+      vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi
+        .fn()
+        .mockResolvedValue([]);
+
+      await publisher.createAndSetupTracks();
+
+      expect(
+        connection.livekitRoom.localParticipant.createTracks,
+      ).toHaveBeenCalledOnce();
+
+      expect(
+        connection.livekitRoom.localParticipant.createTracks,
+      ).toHaveBeenCalledWith({
+        audio: mutes.audioEnabled ? true : undefined,
+        video: mutes.videoEnabled ? true : undefined,
+      });
+    });
+  });
+
+  it("does mute unmute audio", async () => {});
+});
+
+describe("Bug fix", () => {
+  // There is a race condition when creating and publishing tracks while the mute state changes.
+  // This race condition could cause tracks to be published even though they are muted at the
+  // beginning of a call coming from lobby.
+  // This is caused by our stack using manually the low level API to create and publish tracks,
+  // but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create
+  // and publish tracks, and managing pending publications.
+  // Race is as follow, on creation of the Publisher we create the tracks then publish them.
+  // If in the middle of that process the mute state changes:
+  //  - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication
+  //  - If start publication is requested it will publish the track even though there was a mute request.
+  it("wrongly publish tracks while muted", async () => {
+    // setLogLevel(`debug`);
     const publisher = new Publisher(
       scope,
       connection,
@@ -73,68 +327,34 @@ describe("Publisher", () => {
       constant({ supported: false, processor: undefined }),
       logger,
     );
+    audioEnabled$.next(true);
 
-    // should do nothing if no tracks have been created yet.
+    const resolvers = Promise.withResolvers<void>();
-    await publisher.startPublishing();
+    createTrackLock = resolvers.promise;
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).not.toHaveBeenCalled();
 
-    await expect(publisher.createAndSetupTracks()).rejects.toThrow(
+    // Initially the audio is unmuted, so creating tracks should publish the audio track
-      Error("audio and video is false"),
+    const createTracks = publisher.createAndSetupTracks();
-    );
+    void publisher.startPublishing();
-
+    void createTracks.then(() => {
-    (muteStates.audio.enabled$ as BehaviorSubject<boolean>).next(true);
+      void publisher.startPublishing();
-
-    (
-      connection.livekitRoom.localParticipant.createTracks as Mock
-    ).mockResolvedValue([{}, {}]);
-
-    await expect(publisher.createAndSetupTracks()).resolves.not.toThrow();
-    expect(
-      connection.livekitRoom.localParticipant.createTracks,
-    ).toHaveBeenCalledOnce();
-
-    // failiour due to localParticipant.publishTrack
-    (
-      connection.livekitRoom.localParticipant.publishTrack as Mock
-    ).mockRejectedValue(Error("testError"));
-
-    await expect(publisher.startPublishing()).rejects.toThrow(
-      new FailToStartLivekitConnection("testError"),
-    );
-
-    // does not try other conenction after the first one failed
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).toHaveBeenCalledTimes(1);
-
-    // failiour due to connection.state$
-    const beforeState = connection.state$.value;
-    (connection.state$ as BehaviorSubject<ConnectionState>).next({
-      state: "FailedToStart",
-      error: Error("testStartError"),
     });
+    // now mute the audio before allowing track creation to complete
+    audioEnabled$.next(false);
+    resolvers.resolve(undefined);
+    await createTracks;
 
-    await expect(publisher.startPublishing()).rejects.toThrow(
+    await flushPromises();
-      new FailToStartLivekitConnection("testStartError"),
-    );
-    (connection.state$ as BehaviorSubject<ConnectionState>).next(beforeState);
 
-    // does not try other conenction after the first one failed
+    const track = localParticipant.getTrackPublication(
-    expect(
+      Track.Source.Microphone,
-      connection.livekitRoom.localParticipant.publishTrack,
+    )?.track;
-    ).toHaveBeenCalledTimes(1);
+    expect(track).toBeDefined();
 
-    // success case
+    try {
-    (
+      expect(localParticipant.publishTrack).not.toHaveBeenCalled();
-      connection.livekitRoom.localParticipant.publishTrack as Mock
+    } catch {
-    ).mockResolvedValue({});
+      expect(track!.mute).toHaveBeenCalled();
-
+      expect(track!.isMuted).toBe(true);
-    await expect(publisher.startPublishing()).resolves.not.toThrow();
+    }
-
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).toHaveBeenCalledTimes(3);
   });
 });

src/state/CallViewModel/localMember/Publisher.ts

@@ -6,15 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 import {
+  ConnectionState as LivekitConnectionState,
+  type LocalTrackPublication,
   LocalVideoTrack,
+  ParticipantEvent,
   type Room as LivekitRoom,
   Track,
-  type LocalTrack,
-  type LocalTrackPublication,
-  ConnectionState as LivekitConnectionState,
 } from "livekit-client";
 import {
-  BehaviorSubject,
   map,
   NEVER,
   type Observable,
@@ -34,10 +33,6 @@ import { getUrlParams } from "../../../UrlParams.ts";
 import { observeTrackReference$ } from "../../MediaViewModel.ts";
 import { type Connection } from "../remoteMembers/Connection.ts";
 import { type ObservableScope } from "../../ObservableScope.ts";
-import {
-  ElementCallError,
-  FailToStartLivekitConnection,
-} from "../../../utils/errors.ts";
 
 /**
  * A wrapper for a Connection object.
@@ -45,14 +40,21 @@ import {
  * The Publisher is also responsible for creating the media tracks.
  */
 export class Publisher {
+  /**
+   * By default, livekit will start publishing tracks as soon as they are created.
+   * In the matrix RTC world, we want to control when tracks are published based
+   * on whether the user is part of the RTC session or not.
+   */
+  public shouldPublish = false;
+
   /**
    * Creates a new Publisher.
    * @param scope - The observable scope to use for managing the publisher.
    * @param connection - The connection to use for publishing.
    * @param devices - The media devices to use for audio and video input.
    * @param muteStates - The mute states for audio and video.
-   * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
    * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
+   * @param logger - The logger to use for logging :D.
    */
   public constructor(
     private scope: ObservableScope,
@@ -62,7 +64,6 @@ export class Publisher {
     trackerProcessorState$: Behavior<ProcessorState>,
     private logger: Logger,
   ) {
-    this.logger.info("Create LiveKit room");
     const { controlledAudioDevices } = getUrlParams();
 
     const room = connection.livekitRoom;
@@ -80,41 +81,63 @@ export class Publisher {
     this.scope.onEnd(() => {
       this.logger.info("Scope ended -> stop publishing all tracks");
       void this.stopPublishing();
+      muteStates.audio.unsetHandler();
+      muteStates.video.unsetHandler();
     });
 
-    // TODO move mute state handling here using reconcile (instead of inside the mute state class)
+    this.connection.livekitRoom.localParticipant.on(
-    // this.scope.reconcile(
+      ParticipantEvent.LocalTrackPublished,
-    //   this.scope.behavior(
+      this.onLocalTrackPublished.bind(this),
-    //     combineLatest([this.muteStates.video.enabled$, this.tracks$]),
+    );
-    //   ),
-    //   async ([videoEnabled, tracks]) => {
-    //     const track = tracks.find((t) => t.kind == Track.Kind.Video);
-    //     if (!track) return;
-
-    //     if (videoEnabled) {
-    //       await track.unmute();
-    //     } else {
-    //       await track.mute();
-    //     }
-    //   },
-    // );
   }
 
-  private _tracks$ = new BehaviorSubject<LocalTrack<Track.Kind>[]>([]);
+  // LiveKit will publish the tracks as soon as they are created
-  public tracks$ = this._tracks$ as Behavior<LocalTrack<Track.Kind>[]>;
+  // but we want to control when tracks are published.
-
+  // We cannot just mute the tracks, even if this will effectively stop the publishing,
+  // it would also prevent the user from seeing their own video/audio preview.
+  // So for that we use pauseUpStream():  Stops sending media to the server by replacing
+  // the sender track with null, but keeps the local MediaStreamTrack active.
+  // The user can still see/hear themselves locally, but remote participants see nothing.
+  private onLocalTrackPublished(
+    localTrackPublication: LocalTrackPublication,
+  ): void {
+    this.logger.info("Local track published", localTrackPublication);
+    const lkRoom = this.connection.livekitRoom;
+    if (!this.shouldPublish) {
+      this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => {
+        this.logger.error(`Failed to pause upstreams`, e);
+      });
+    }
+    // also check the mute state and apply it
+    if (localTrackPublication.source === Track.Source.Microphone) {
+      const enabled = this.muteStates.audio.enabled$.value;
+      lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => {
+        this.logger.error(
+          `Failed to enable microphone track, enabled:${enabled}`,
+          e,
+        );
+      });
+    } else if (localTrackPublication.source === Track.Source.Camera) {
+      const enabled = this.muteStates.video.enabled$.value;
+      lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => {
+        this.logger.error(
+          `Failed to enable camera track, enabled:${enabled}`,
+          e,
+        );
+      });
+    }
+  }
   /**
-   * Start the connection to LiveKit and publish local tracks.
+   * Create and setup local audio and video tracks based on the current mute states.
+   * It creates the tracks only if audio and/or video is enabled, to avoid unnecessary
+   * permission prompts.
    *
-   * This will:
+   * It also observes mute state changes to update LiveKit microphone/camera states accordingly.
-   * wait for the connection to be ready.
+   * If a track is not created initially because disabled, it will be created when unmuting.
-   // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
+   *
-   // * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
+   * This call is not blocking anymore, instead callers can listen to the
-   // * 3. Connect to the configured LiveKit room.
+   * `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors.
-   // * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
    *
-   * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
-   * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
    */
   public async createAndSetupTracks(): Promise<void> {
     this.logger.debug("createAndSetupTracks called");
@@ -122,119 +145,121 @@ export class Publisher {
     // Observe mute state changes and update LiveKit microphone/camera states accordingly
     this.observeMuteStates(this.scope);
 
-    // TODO-MULTI-SFU: Prepublish a microphone track
+    // Check if audio and/or video is enabled. We only create tracks if enabled,
+    // because it could prompt for permission, and we don't want to do that unnecessarily.
     const audio = this.muteStates.audio.enabled$.value;
     const video = this.muteStates.video.enabled$.value;
-    // createTracks throws if called with audio=false and video=false
+
-    if (audio || video) {
+    // We don't await the creation, because livekit could block until the tracks
-      // TODO this can still throw errors? It will also prompt for permissions if not already granted
+    // are fully published, and not only that they are created.
-      return lkRoom.localParticipant
+    // We don't have control on that, localParticipant creates and publishes the tracks
-        .createTracks({
+    // asap.
-          audio,
+    // We are using the `ParticipantEvent.LocalTrackPublished` to be notified
-          video,
+    // when tracks are actually published, and at that point
-        })
+    // we can pause upstream if needed (depending on if startPublishing has been called).
-        .then((tracks) => {
+    if (audio && video) {
-          this.logger.info(
+      // Enable both at once in order to have a single permission prompt!
-            "created track",
+      void lkRoom.localParticipant.enableCameraAndMicrophone();
-            tracks.map((t) => t.kind + ", " + t.id),
+    } else if (audio) {
-          );
+      void lkRoom.localParticipant.setMicrophoneEnabled(true);
-          this._tracks$.next(tracks);
+    } else if (video) {
-        })
+      void lkRoom.localParticipant.setCameraEnabled(true);
-        .catch((error) => {
+    }
-          this.logger.error("Failed to create tracks", error);
+
-        });
+    return Promise.resolve();
-    }
+  }
-    throw Error("audio and video is false");
+
+  private async pauseUpstreams(
+    lkRoom: LivekitRoom,
+    sources: Track.Source[],
+  ): Promise<void> {
+    for (const source of sources) {
+      const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
+      if (track) {
+        await track.pauseUpstream();
+      } else {
+        this.logger.warn(
+          `No track found for source ${source} to pause upstream`,
+        );
+      }
+    }
+  }
+
+  private async resumeUpstreams(
+    lkRoom: LivekitRoom,
+    sources: Track.Source[],
+  ): Promise<void> {
+    for (const source of sources) {
+      const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
+      if (track) {
+        await track.resumeUpstream();
+      } else {
+        this.logger.warn(
+          `No track found for source ${source} to resume upstream`,
+        );
+      }
+    }
   }
 
-  private _publishing$ = new BehaviorSubject<boolean>(false);
-  public publishing$ = this.scope.behavior(this._publishing$);
   /**
+   *
+   * Request to publish local tracks to the LiveKit room.
+   * This will wait for the connection to be ready before publishing.
+   * Livekit also have some local retry logic for publishing tracks.
+   * Can be called multiple times, localparticipant manages the state of published tracks (or pending publications).
    *
    * @returns
-   * @throws ElementCallError
    */
-  public async startPublishing(): Promise<LocalTrack[]> {
+  public async startPublishing(): Promise<void> {
+    if (this.shouldPublish) {
+      this.logger.debug(`Already publishing, ignoring startPublishing call`);
+      return;
+    }
+    this.shouldPublish = true;
     this.logger.debug("startPublishing called");
+
     const lkRoom = this.connection.livekitRoom;
-    const { promise, resolve, reject } = Promise.withResolvers<void>();
+
-    const sub = this.connection.state$.subscribe((s) => {
+    // Resume upstream for both audio and video tracks
-      switch (s.state) {
+    // We need to call it explicitly because call setTrackEnabled does not always
-        case "ConnectedToLkRoom":
+    // resume upstream. It will only if you switch the track from disabled to enabled,
-          resolve();
+    // but if the track is already enabled but upstream is paused, it won't resume it.
-          break;
+    // TODO what about screen share?
-        case "FailedToStart":
-          reject(
-            s.error instanceof ElementCallError
-              ? s.error
-              : new FailToStartLivekitConnection(s.error.message),
-          );
-          break;
-        default:
-          this.logger.info("waiting for connection: ", s.state);
-      }
-    });
     try {
-      await promise;
+      await this.resumeUpstreams(lkRoom, [
+        Track.Source.Microphone,
+        Track.Source.Camera,
+      ]);
     } catch (e) {
-      throw e;
+      this.logger.error(`Failed to resume upstreams`, e);
-    } finally {
-      sub.unsubscribe();
     }
-
-    for (const track of this.tracks$.value) {
-      this.logger.info("publish ", this.tracks$.value.length, "tracks");
-      // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
-      // with a timeout.
-      await lkRoom.localParticipant.publishTrack(track).catch((error) => {
-        this.logger.error("Failed to publish track", error);
-        throw new FailToStartLivekitConnection(
-          error instanceof Error ? error.message : error,
-        );
-      });
-      this.logger.info("published track ", track.kind, track.id);
-
-      // TODO: check if the connection is still active? and break the loop if not?
-    }
-    this._publishing$.next(true);
-    return this.tracks$.value;
   }
 
   public async stopPublishing(): Promise<void> {
     this.logger.debug("stopPublishing called");
-    // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
+    this.shouldPublish = false;
-    // actually has the right lifetime
+    // Pause upstream will stop sending media to the server, while keeping
-    this.muteStates.audio.unsetHandler();
+    // the local MediaStreamTrack active, so the user can still see themselves.
-    this.muteStates.video.unsetHandler();
+    await this.pauseUpstreams(this.connection.livekitRoom, [
-
+      Track.Source.Microphone,
-    const localParticipant = this.connection.livekitRoom.localParticipant;
+      Track.Source.Camera,
-    const tracks: LocalTrack[] = [];
+      Track.Source.ScreenShare,
-    const addToTracksIfDefined = (p: LocalTrackPublication): void => {
+    ]);
-      if (p.track !== undefined) tracks.push(p.track);
-    };
-    localParticipant.trackPublications.forEach(addToTracksIfDefined);
-    this.logger.debug(
-      "list of tracks to unpublish:",
-      tracks.map((t) => t.kind + ", " + t.id),
-      "start unpublishing now",
-    );
-    await localParticipant.unpublishTracks(tracks).catch((error) => {
-      this.logger.error("Failed to unpublish tracks", error);
-      throw error;
-    });
-    this.logger.debug(
-      "unpublished tracks",
-      tracks.map((t) => t.kind + ", " + t.id),
-    );
-    this._publishing$.next(false);
   }
 
-  /**
+  public async stopTracks(): Promise<void> {
-   * Stops all tracks that are currently running
+    const lkRoom = this.connection.livekitRoom;
-   */
+    for (const source of [
-  public stopTracks(): void {
+      Track.Source.Microphone,
-    this.tracks$.value.forEach((t) => t.stop());
+      Track.Source.Camera,
-    this._tracks$.next([]);
+      Track.Source.ScreenShare,
+    ]) {
+      const localPub = lkRoom.localParticipant.getTrackPublication(source);
+      if (localPub?.track) {
+        // stops and unpublishes the track
+        await lkRoom.localParticipant.unpublishTrack(localPub!.track, true);
+      }
+    }
   }
 
   /// Private methods
@@ -336,17 +361,31 @@ export class Publisher {
    */
   private observeMuteStates(scope: ObservableScope): void {
     const lkRoom = this.connection.livekitRoom;
-    this.muteStates.audio.setHandler(async (desired) => {
+    this.muteStates.audio.setHandler(async (enable) => {
       try {
-        await lkRoom.localParticipant.setMicrophoneEnabled(desired);
+        this.logger.debug(
+          `handler: Setting LiveKit microphone enabled: ${enable}`,
+        );
+        await lkRoom.localParticipant.setMicrophoneEnabled(enable);
+        // Unmute will restart the track if it was paused upstream,
+        // but until explicitly requested, we want to keep it paused.
+        if (!this.shouldPublish && enable) {
+          await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
+        }
       } catch (e) {
         this.logger.error("Failed to update LiveKit audio input mute state", e);
       }
       return lkRoom.localParticipant.isMicrophoneEnabled;
     });
-    this.muteStates.video.setHandler(async (desired) => {
+    this.muteStates.video.setHandler(async (enable) => {
       try {
-        await lkRoom.localParticipant.setCameraEnabled(desired);
+        this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
+        await lkRoom.localParticipant.setCameraEnabled(enable);
+        // Unmute will restart the track if it was paused upstream,
+        // but until explicitly requested, we want to keep it paused.
+        if (!this.shouldPublish && enable) {
+          await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
+        }
       } catch (e) {
         this.logger.error("Failed to update LiveKit video input mute state", e);
       }
@@ -362,7 +401,7 @@ export class Publisher {
     const track$ = scope.behavior(
       observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
         map((trackRef) => {
-          const track = trackRef?.publication?.track;
+          const track = trackRef?.publication.track;
           return track instanceof LocalVideoTrack ? track : null;
         }),
       ),

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -30,13 +30,16 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import {
   Connection,
+  ConnectionState,
   type ConnectionOpts,
-  type ConnectionState,
-  type PublishingParticipant,
 } from "./Connection.ts";
 import { ObservableScope } from "../../ObservableScope.ts";
 import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
-import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
+import {
+  ElementCallError,
+  FailToGetOpenIdToken,
+} from "../../../utils/errors.ts";
+import { mockRemoteParticipant } from "../../../utils/test.ts";
 
 let testScope: ObservableScope;
 
@@ -47,11 +50,6 @@ let fakeLivekitRoom: MockedObject<LivekitRoom>;
 let localParticipantEventEmiter: EventEmitter;
 let fakeLocalParticipant: MockedObject<LocalParticipant>;
 
-let fakeRoomEventEmiter: EventEmitter;
-// let fakeMembershipsFocusMap$: BehaviorSubject<
-//   { membership: CallMembership; transport: LivekitTransport }[]
-// >;
-
 const livekitFocus: LivekitTransport = {
   livekit_alias: "!roomID:example.org",
   livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
@@ -88,22 +86,25 @@ function setupTest(): void {
       localParticipantEventEmiter,
     ),
   } as unknown as LocalParticipant);
-  fakeRoomEventEmiter = new EventEmitter();
 
+  const fakeRoomEventEmitter = new EventEmitter();
   fakeLivekitRoom = vi.mocked<LivekitRoom>({
     connect: vi.fn(),
     disconnect: vi.fn(),
     remoteParticipants: new Map(),
     localParticipant: fakeLocalParticipant,
     state: LivekitConnectionState.Disconnected,
-    on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
+    on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter),
-    off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
+    off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter),
-    addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
+    addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter),
     removeListener:
-      fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
+      fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter),
     removeAllListeners:
-      fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
+      fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter),
     setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
+    emit: (eventName: string | symbol, ...args: unknown[]) => {
+      fakeRoomEventEmitter.emit(eventName, ...args);
+    },
   } as unknown as LivekitRoom);
 }
 
@@ -125,7 +126,16 @@ function setupRemoteConnection(): Connection {
     };
   });
 
-  fakeLivekitRoom.connect.mockResolvedValue(undefined);
+  fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
+    const changeEv = RoomEvent.ConnectionStateChanged;
+
+    fakeLivekitRoom.state = LivekitConnectionState.Connecting;
+    fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
+    fakeLivekitRoom.state = LivekitConnectionState.Connected;
+    fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
+
+    return Promise.resolve();
+  });
 
   return new Connection(opts, logger);
 }
@@ -148,7 +158,7 @@ describe("Start connection states", () => {
     };
     const connection = new Connection(opts, logger);
 
-    expect(connection.state$.getValue().state).toEqual("Initialized");
+    expect(connection.state$.getValue()).toEqual("Initialized");
   });
 
   it("fail to getOpenId token then error state", async () => {
@@ -164,7 +174,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -184,22 +194,20 @@ describe("Start connection states", () => {
 
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
-    expect(capturedState!.state).toEqual("FetchingConfig");
+    expect(capturedState!).toEqual("FetchingConfig");
 
     deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
 
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
-    if (capturedState!.state === "FailedToStart") {
+    if (capturedState instanceof Error) {
-      expect(capturedState!.error.message).toEqual("Something went wrong");
+      expect(capturedState.message).toEqual("Something went wrong");
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
       );
     } else {
-      expect.fail(
+      expect.fail("Expected FailedToStart state but got " + capturedState);
-        "Expected FailedToStart state but got " + capturedState?.state,
-      );
     }
   });
 
@@ -216,7 +224,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -238,24 +246,25 @@ describe("Start connection states", () => {
 
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
-    expect(capturedState?.state).toEqual("FetchingConfig");
+    expect(capturedState).toEqual(ConnectionState.FetchingConfig);
 
     deferredSFU.resolve();
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
 
-    if (capturedState?.state === "FailedToStart") {
+    if (
-      expect(capturedState?.error.message).toContain(
+      capturedState instanceof ElementCallError &&
+      capturedState.cause instanceof Error
+    ) {
+      expect(capturedState.cause.message).toContain(
         "SFU Config fetch failed with exception Error",
       );
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
       );
     } else {
-      expect.fail(
+      expect.fail("Expected FailedToStart state but got " + capturedState);
-        "Expected FailedToStart state but got " + capturedState?.state,
-      );
     }
   });
 
@@ -272,7 +281,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -302,15 +311,18 @@ describe("Start connection states", () => {
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
 
-    expect(capturedState?.state).toEqual("FetchingConfig");
+    expect(capturedState).toEqual(ConnectionState.FetchingConfig);
 
     deferredSFU.resolve();
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
 
-    if (capturedState && capturedState?.state === "FailedToStart") {
+    if (
-      expect(capturedState.error.message).toContain(
+      capturedState instanceof ElementCallError &&
+      capturedState.cause instanceof Error
+    ) {
+      expect(capturedState.cause.message).toContain(
         "Failed to connect to livekit",
       );
       expect(connection.transport.livekit_alias).toEqual(
@@ -329,7 +341,7 @@ describe("Start connection states", () => {
 
     const connection = setupRemoteConnection();
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -339,13 +351,15 @@ describe("Start connection states", () => {
     await vi.runAllTimersAsync();
 
     const initialState = capturedStates.shift();
-    expect(initialState?.state).toEqual("Initialized");
+    expect(initialState).toEqual(ConnectionState.Initialized);
     const fetchingState = capturedStates.shift();
-    expect(fetchingState?.state).toEqual("FetchingConfig");
+    expect(fetchingState).toEqual(ConnectionState.FetchingConfig);
+    const disconnectedState = capturedStates.shift();
+    expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected);
     const connectingState = capturedStates.shift();
-    expect(connectingState?.state).toEqual("ConnectingToLkRoom");
+    expect(connectingState).toEqual(ConnectionState.LivekitConnecting);
     const connectedState = capturedStates.shift();
-    expect(connectedState?.state).toEqual("ConnectedToLkRoom");
+    expect(connectedState).toEqual(ConnectionState.LivekitConnected);
   });
 
   it("shutting down the scope should stop the connection", async () => {
@@ -363,44 +377,32 @@ describe("Start connection states", () => {
   });
 });
 
-function fakeRemoteLivekitParticipant(
+describe("remote participants", () => {
-  id: string,
+  it("emits the list of remote participants", () => {
-  publications: number = 1,
-): RemoteParticipant {
-  return {
-    identity: id,
-    getTrackPublications: () => Array(publications),
-  } as unknown as RemoteParticipant;
-}
-
-describe("Publishing participants observations", () => {
-  it("should emit the list of publishing participants", () => {
     setupTest();
 
     const connection = setupRemoteConnection();
 
-    const bobIsAPublisher = Promise.withResolvers<void>();
+    const observedParticipants: RemoteParticipant[][] = [];
-    const danIsAPublisher = Promise.withResolvers<void>();
+    const s = connection.remoteParticipants$.subscribe((participants) => {
-    const observedPublishers: PublishingParticipant[][] = [];
+      observedParticipants.push(participants);
-    const s = connection.remoteParticipants$.subscribe((publishers) => {
-      observedPublishers.push(publishers);
-      if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
-        bobIsAPublisher.resolve();
-      }
-      if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
-        danIsAPublisher.resolve();
-      }
     });
     onTestFinished(() => s.unsubscribe());
-    // The publishingParticipants$ observable is derived from the current members of the
+    // The remoteParticipants$ observable is derived from the current members of the
     // livekitRoom and the rtc membership in order to publish the members that are publishing
     // on this connection.
 
-    let participants: RemoteParticipant[] = [
+    const participants: RemoteParticipant[] = [
-      fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0),
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
-      fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
-      fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0),
+      // Mock Dan to have no published tracks. We want him to still show show up
+      // in the participants list.
+      mockRemoteParticipant({
+        identity: "@dan:example.org:DEV333",
+        getTrackPublication: () => undefined,
+        getTrackPublications: () => [],
+      }),
     ];
 
     // Let's simulate 3 members on the livekitRoom
@@ -409,9 +411,10 @@ describe("Publishing participants observations", () => {
     );
 
     participants.forEach((p) =>
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
+<<<<<<< HEAD
     // At this point there should be ~~no~~ publishers
     // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
     // The filtering we do is just based on the matrixRTC member events.
@@ -429,6 +432,10 @@ describe("Publishing participants observations", () => {
 
     // At this point there should be no publishers
     expect(observedPublishers.pop()!.length).toEqual(4);
+=======
+    // All remote participants should be present
+    expect(observedParticipants.pop()!.length).toEqual(4);
+>>>>>>> livekit
   });
 
   it("should be scoped to parent scope", (): void => {
@@ -436,14 +443,20 @@ describe("Publishing participants observations", () => {
 
     const connection = setupRemoteConnection();
 
+<<<<<<< HEAD
     let observedPublishers: PublishingParticipant[][] = [];
     const s = connection.remoteParticipants$.subscribe((publishers) => {
       observedPublishers.push(publishers);
+=======
+    let observedParticipants: RemoteParticipant[][] = [];
+    const s = connection.remoteParticipants$.subscribe((participants) => {
+      observedParticipants.push(participants);
+>>>>>>> livekit
     });
     onTestFinished(() => s.unsubscribe());
 
     let participants: RemoteParticipant[] = [
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
     ];
 
     // Let's simulate 3 members on the livekitRoom
@@ -452,9 +465,10 @@ describe("Publishing participants observations", () => {
     );
 
     for (const participant of participants) {
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
     }
 
+<<<<<<< HEAD
     // At this point there should be ~~no~~ publishers
     // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
     // The filtering we do is just based on the matrixRTC member events.
@@ -470,22 +484,28 @@ describe("Publishing participants observations", () => {
     const publishers = observedPublishers.pop();
     expect(publishers?.length).toEqual(1);
     expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
+=======
+    // We should have bob as a participant now
+    const ps = observedParticipants.pop();
+    expect(ps?.length).toEqual(1);
+    expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
+>>>>>>> livekit
 
     // end the parent scope
     testScope.end();
-    observedPublishers = [];
+    observedParticipants = [];
 
-    // SHOULD NOT emit any more publishers as the scope is ended
+    // SHOULD NOT emit any more participants as the scope is ended
     participants = participants.filter(
       (p) => p.identity !== "@bob:example.org:DEV111",
     );
 
-    fakeRoomEventEmiter.emit(
+    fakeLivekitRoom.emit(
       RoomEvent.ParticipantDisconnected,
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
     );
 
-    expect(observedPublishers.length).toEqual(0);
+    expect(observedParticipants.length).toEqual(0);
   });
 });
 

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -12,11 +12,8 @@ import {
 } from "@livekit/components-core";
 import {
   ConnectionError,
-  type ConnectionState as LivekitConenctionState,
   type Room as LivekitRoom,
-  type LocalParticipant,
   type RemoteParticipant,
-  RoomEvent,
 } from "livekit-client";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { BehaviorSubject } from "rxjs";
@@ -30,12 +27,12 @@ import {
 import { type Behavior } from "../../Behavior.ts";
 import { type ObservableScope } from "../../ObservableScope.ts";
 import {
+  ElementCallError,
   InsufficientCapacityError,
   SFURoomCreationRestrictedError,
+  UnknownCallError,
 } from "../../../utils/errors.ts";
 
-export type PublishingParticipant = LocalParticipant | RemoteParticipant;
-
 export interface ConnectionOpts {
   /** The media transport to connect to. */
   transport: LivekitTransport;
@@ -47,17 +44,30 @@ export interface ConnectionOpts {
   /** Optional factory to create the LiveKit room, mainly for testing purposes. */
   livekitRoomFactory: () => LivekitRoom;
 }
-
+export class FailedToStartError extends Error {
-export type ConnectionState =
+  public constructor(message: string) {
-  | { state: "Initialized" }
+    super(message);
-  | { state: "FetchingConfig" }
+    this.name = "FailedToStartError";
-  | { state: "ConnectingToLkRoom" }
+  }
-  | {
+}
-      state: "ConnectedToLkRoom";
+
-      livekitConnectionState$: Behavior<LivekitConenctionState>;
+export enum ConnectionState {
+  /** The start state of a connection. It has been created but nothing has loaded yet. */
+  Initialized = "Initialized",
+  /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room  */
+  FetchingConfig = "FetchingConfig",
+  Stopped = "Stopped",
+  /** The same as ConnectionState.Disconnected from `livekit-client` */
+  LivekitDisconnected = "disconnected",
+  /** The same as ConnectionState.Connecting from `livekit-client` */
+  LivekitConnecting = "connecting",
+  /** The same as ConnectionState.Connected from `livekit-client` */
+  LivekitConnected = "connected",
+  /** The same as ConnectionState.Reconnecting from `livekit-client` */
+  LivekitReconnecting = "reconnecting",
+  /** The same as ConnectionState.SignalReconnecting from `livekit-client` */
+  LivekitSignalReconnecting = "signalReconnecting",
 }
-  | { state: "FailedToStart"; error: Error }
-  | { state: "Stopped" };
 
 /**
  * A connection to a Matrix RTC LiveKit backend.
@@ -66,14 +76,14 @@ export type ConnectionState =
  */
 export class Connection {
   // Private Behavior
-  private readonly _state$ = new BehaviorSubject<ConnectionState>({
+  private readonly _state$ = new BehaviorSubject<
-    state: "Initialized",
+    ConnectionState | ElementCallError
-  });
+  >(ConnectionState.Initialized);
 
   /**
    * The current state of the connection to the media transport.
    */
-  public readonly state$: Behavior<ConnectionState> = this._state$;
+  public readonly state$: Behavior<ConnectionState | Error> = this._state$;
 
   /**
    * The media transport to connect to.
@@ -85,11 +95,13 @@ export class Connection {
   private scope: ObservableScope;
 
   /**
-   * An observable of the participants that are publishing on this connection. (Excluding our local participant)
+   * The remote LiveKit participants that are visible on this connection.
-   * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
+   *
-   * It filters the participants to only those that are associated with a membership that claims to publish on this connection.
+   * Note that this may include participants that are connected only to
+   * subscribe, or publishers that are otherwise unattested in MatrixRTC state.
+   * It is therefore more low-level than what should be presented to the user.
    */
-  public readonly remoteParticipants$: Behavior<PublishingParticipant[]>;
+  public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
 
   /**
    * Whether the connection has been stopped.
@@ -115,16 +127,24 @@ export class Connection {
     this.logger.debug("Starting Connection");
     this.stopped = false;
     try {
-      this._state$.next({
+      this._state$.next(ConnectionState.FetchingConfig);
-        state: "FetchingConfig",
+      // We should already have this information after creating the localTransport.
-      });
+      // It would probably be better to forward this here.
       const { url, jwt } = await this.getSFUConfigWithOpenID();
       // If we were stopped while fetching the config, don't proceed to connect
       if (this.stopped) return;
 
-      this._state$.next({
+      // Setup observer once we are done with getSFUConfigWithOpenID
-        state: "ConnectingToLkRoom",
+      connectionStateObserver(this.livekitRoom)
+        .pipe(
+          this.scope.bind(),
+          map((s) => s as unknown as ConnectionState),
+        )
+        .subscribe((lkState) => {
+          // It is save to cast lkState to ConnectionState as they are fully overlapping.
+          this._state$.next(lkState);
         });
+
       try {
         await this.livekitRoom.connect(url, jwt);
       } catch (e) {
@@ -139,7 +159,8 @@ export class Connection {
             throw new InsufficientCapacityError();
           }
           if (e.status === 404) {
-            // error msg is "Could not establish signal connection: requested room does not exist"
+            // error msg is "Failed to create call"
+            // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists."
             // The room does not exist. There are two different modes of operation for the SFU:
             // - the room is created on the fly when connecting (livekit `auto_create` option)
             // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
@@ -151,23 +172,16 @@ export class Connection {
       }
       // If we were stopped while connecting, don't proceed to update state.
       if (this.stopped) return;
-
-      this._state$.next({
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: this.scope.behavior(
-          connectionStateObserver(this.livekitRoom),
-        ),
-      });
-      this.logger.info(
-        "Connected to LiveKit room",
-        this.transport.livekit_service_url,
-      );
     } catch (error) {
       this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
-      this._state$.next({
+      this._state$.next(
-        state: "FailedToStart",
+        error instanceof ElementCallError
-        error: error instanceof Error ? error : new Error(`${error}`),
+          ? error
-      });
+          : error instanceof Error
+            ? new UnknownCallError(error)
+            : new UnknownCallError(new Error(`${error}`)),
+      );
+      // Its okay to ignore the throw. The error is part of the state.
       throw error;
     }
   }
@@ -192,9 +206,7 @@ export class Connection {
     );
     if (this.stopped) return;
     await this.livekitRoom.disconnect();
-    this._state$.next({
+    this._state$.next(ConnectionState.Stopped);
-      state: "Stopped",
-    });
     this.stopped = true;
   }
 
@@ -220,24 +232,9 @@ export class Connection {
     this.transport = transport;
     this.client = client;
 
-    // REMOTE participants with track!!!
-    // this.remoteParticipants$
     this.remoteParticipants$ = scope.behavior(
-      // only tracks remote participants
+      // Only tracks remote participants
-      connectedParticipantsObserver(this.livekitRoom, {
+      connectedParticipantsObserver(this.livekitRoom),
-        additionalRoomEvents: [
-          RoomEvent.TrackPublished,
-          RoomEvent.TrackUnpublished,
-        ],
-      }),
-      // .pipe(
-      //   map((participants) => {
-      //     return participants.filter(
-      //       (participant) => participant.getTrackPublications().length > 0,
-      //     );
-      //   }),
-      // )
-      [],
     );
 
     scope.onEnd(() => {

src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
 import { BehaviorSubject } from "rxjs";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
-import { type Participant as LivekitParticipant } from "livekit-client";
+import { type RemoteParticipant } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
@@ -201,23 +201,20 @@ describe("connections$ stream", () => {
 
 describe("connectionManagerData$ stream", () => {
   // Used in test to control fake connections' remoteParticipants$ streams
-  let fakePublishingParticipantsStreams: Map<
+  let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
-    string,
-    Behavior<LivekitParticipant[]>
-  >;
 
   function keyForTransport(transport: LivekitTransport): string {
     return `${transport.livekit_service_url}|${transport.livekit_alias}`;
   }
 
   beforeEach(() => {
-    fakePublishingParticipantsStreams = new Map();
+    fakeRemoteParticipantsStreams = new Map();
 
-    function getPublishingParticipantsFor(
+    function getRemoteParticipantsFor(
       transport: LivekitTransport,
-    ): Behavior<LivekitParticipant[]> {
+    ): Behavior<RemoteParticipant[]> {
       return (
-        fakePublishingParticipantsStreams.get(keyForTransport(transport)) ??
+        fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
         new BehaviorSubject([])
       );
     }
@@ -227,12 +224,12 @@ describe("connectionManagerData$ stream", () => {
       .fn()
       .mockImplementation(
         (transport: LivekitTransport, scope: ObservableScope) => {
-          const fakePublishingParticipants$ = new BehaviorSubject<
+          const fakeRemoteParticipants$ = new BehaviorSubject<
-            LivekitParticipant[]
+            RemoteParticipant[]
           >([]);
           const mockConnection = {
             transport,
-            remoteParticipants$: getPublishingParticipantsFor(transport),
+            remoteParticipants$: getRemoteParticipantsFor(transport),
           } as unknown as Connection;
           vi.mocked(mockConnection).start = vi.fn();
           vi.mocked(mockConnection).stop = vi.fn();
@@ -241,36 +238,36 @@ describe("connectionManagerData$ stream", () => {
             void mockConnection.stop();
           });
 
-          fakePublishingParticipantsStreams.set(
+          fakeRemoteParticipantsStreams.set(
             keyForTransport(transport),
-            fakePublishingParticipants$,
+            fakeRemoteParticipants$,
           );
           return mockConnection;
         },
       );
   });
 
-  test("Should report connections with the publishing participants", () => {
+  test("Should report connections with the remote participants", () => {
     withTestScheduler(({ expectObservable, schedule, behavior }) => {
       // Setup the fake participants streams behavior
       // ==============================
-      fakePublishingParticipantsStreams.set(
+      fakeRemoteParticipantsStreams.set(
         keyForTransport(TRANSPORT_1),
         behavior("oa-b", {
           o: [],
-          a: [{ identity: "user1A" } as LivekitParticipant],
+          a: [{ identity: "user1A" } as RemoteParticipant],
           b: [
-            { identity: "user1A" } as LivekitParticipant,
+            { identity: "user1A" } as RemoteParticipant,
-            { identity: "user1B" } as LivekitParticipant,
+            { identity: "user1B" } as RemoteParticipant,
           ],
         }),
       );
 
-      fakePublishingParticipantsStreams.set(
+      fakeRemoteParticipantsStreams.set(
         keyForTransport(TRANSPORT_2),
         behavior("o-a", {
           o: [],
-          a: [{ identity: "user2A" } as LivekitParticipant],
+          a: [{ identity: "user2A" } as RemoteParticipant],
         }),
       );
       // ==============================

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
+import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
-  type LivekitTransport,
-  type ParticipantId,
-} from "matrix-js-sdk/lib/matrixrtc";
 import { combineLatest, map, of, switchMap, tap } from "rxjs";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
+import { type RemoteParticipant } from "livekit-client";
 
 import { type Behavior } from "../../Behavior.ts";
 import { type Connection } from "./Connection.ts";
@@ -22,20 +19,12 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 
 export class ConnectionManagerData {
-  private readonly store: Map<
+  private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
-    string,
+    new Map();
-    {
-      connection: Connection;
-      participants: (LocalParticipant | RemoteParticipant)[];
-    }
-  > = new Map();
 
   public constructor() {}
 
-  public add(
+  public add(connection: Connection, participants: RemoteParticipant[]): void {
-    connection: Connection,
-    participants: (LocalParticipant | RemoteParticipant)[],
-  ): void {
     const key = this.getKey(connection.transport);
     const existing = this.store.get(key);
     if (!existing) {
@@ -61,7 +50,7 @@ export class ConnectionManagerData {
 
   public getParticipantsForTransport(
     transport: LivekitTransport,
-  ): (LocalParticipant | RemoteParticipant)[] {
+  ): RemoteParticipant[] {
     const key = transport.livekit_service_url + "|" + transport.livekit_alias;
     const existing = this.store.get(key);
     if (existing) {
@@ -69,39 +58,20 @@ export class ConnectionManagerData {
     }
     return [];
   }
+}
 
-  /**
-   * Get all connections where the given participant is publishing.
-   * In theory, there could be several connections where the same participant is publishing but with
-   * only well behaving clients a participant should only be publishing on a single connection.
-   * @param participantId
-   */
-  public getConnectionsForParticipant(
-    participantId: ParticipantId,
-  ): Connection[] {
-    const connections: Connection[] = [];
-    for (const { connection, participants } of this.store.values()) {
-      if (
-        participants.some(
-          (participant) => participant?.identity === participantId,
-        )
-      ) {
-        connections.push(connection);
-      }
-    }
-    return connections;
-  }
-}
 interface Props {
   scope: ObservableScope;
   connectionFactory: ConnectionFactory;
   inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
   logger: Logger;
 }
+
 // TODO - write test for scopes (do we really need to bind scope)
 export interface IConnectionManager {
   connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
 }
+
 /**
  * Crete a `ConnectionManager`
  * @param scope the observable scope used by this object.
@@ -184,7 +154,7 @@ export function createConnectionManager$({
         const epoch = connections.epoch;
 
         // Map the connections to list of {connection, participants}[]
-        const listOfConnectionsWithParticipants = connections.value.map(
+        const listOfConnectionsWithRemoteParticipants = connections.value.map(
           (connection) => {
             return connection.remoteParticipants$.pipe(
               map((participants) => ({
@@ -196,12 +166,16 @@ export function createConnectionManager$({
         );
 
         // probably not required
+<<<<<<< HEAD
         if (listOfConnectionsWithParticipants.length === 0) {
+=======
+        if (listOfConnectionsWithRemoteParticipants.length === 0) {
+>>>>>>> livekit
           return of(new Epoch(new ConnectionManagerData(), epoch));
         }
 
         // combineLatest the several streams into a single stream with the ConnectionManagerData
-        return combineLatest(listOfConnectionsWithParticipants).pipe(
+        return combineLatest(listOfConnectionsWithRemoteParticipants).pipe(
           map(
             (lists) =>
               new Epoch(

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs";
 
 import { type IConnectionManager } from "./ConnectionManager.ts";
 import {
-  type MatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
   createMatrixLivekitMembers$,
 } from "./MatrixLivekitMembers.ts";
 import {
@@ -99,15 +99,13 @@ test("should signal participant not yet connected to livekit", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      "a",
+      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-      {
-        a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
         expect(data.length).toEqual(1);
         expectObservable(data[0].membership$).toBe("a", {
           a: bobMembership,
         });
-          expectObservable(data[0].participant$).toBe("a", {
+        expectObservable(data[0].participant.value$).toBe("a", {
           a: null,
         });
         expectObservable(data[0].connection$).toBe("a", {
@@ -115,8 +113,7 @@ test("should signal participant not yet connected to livekit", () => {
         });
         return true;
       }),
-      },
+    });
-    );
   });
 });
 
@@ -182,15 +179,13 @@ test("should signal participant on a connection that is publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      "a",
+      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-      {
-        a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
         expect(data.length).toEqual(1);
         expectObservable(data[0].membership$).toBe("a", {
           a: bobMembership,
         });
-          expectObservable(data[0].participant$).toBe("a", {
+        expectObservable(data[0].participant.value$).toBe("a", {
           a: expect.toSatisfy((participant) => {
             expect(participant).toBeDefined();
             expect(participant!.identity).toEqual(bobParticipantId);
@@ -202,8 +197,7 @@ test("should signal participant on a connection that is publishing", () => {
         });
         return true;
       }),
-      },
+    });
-    );
   });
 });
 
@@ -236,15 +230,13 @@ test("should signal participant on a connection that is not publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      "a",
+      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-      {
-        a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
         expect(data.length).toEqual(1);
         expectObservable(data[0].membership$).toBe("a", {
           a: bobMembership,
         });
-          expectObservable(data[0].participant$).toBe("a", {
+        expectObservable(data[0].participant.value$).toBe("a", {
           a: null,
         });
         expectObservable(data[0].connection$).toBe("a", {
@@ -252,8 +244,7 @@ test("should signal participant on a connection that is not publishing", () => {
         });
         return true;
       }),
-      },
+    });
-    );
   });
 });
 
@@ -305,7 +296,7 @@ describe("Publication edge case", () => {
       expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
         "a",
         {
-          a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
+          a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
             expect(data.length).toEqual(2);
             expectObservable(data[0].membership$).toBe("a", {
               a: bobMembership,
@@ -314,7 +305,7 @@ describe("Publication edge case", () => {
               // The real connection should be from transportA as per the membership
               a: connectionA,
             });
-            expectObservable(data[0].participant$).toBe("a", {
+            expectObservable(data[0].participant.value$).toBe("a", {
               a: expect.toSatisfy((participant) => {
                 expect(participant).toBeDefined();
                 expect(participant!.identity).toEqual(bobParticipantId);
@@ -371,7 +362,7 @@ describe("Publication edge case", () => {
       expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
         "a",
         {
-          a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
+          a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
             expect(data.length).toEqual(2);
             expectObservable(data[0].membership$).toBe("a", {
               a: bobMembership,
@@ -380,7 +371,7 @@ describe("Publication edge case", () => {
               // The real connection should be from transportA as per the membership
               a: connectionA,
             });
-            expectObservable(data[0].participant$).toBe("a", {
+            expectObservable(data[0].participant.value$).toBe("a", {
               // No participant as Bob is not publishing on his membership transport
               a: null,
             });

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
+import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
-  type LocalParticipant as LocalLivekitParticipant,
-  type RemoteParticipant as RemoteLivekitParticipant,
-} from "livekit-client";
 import {
   type LivekitTransport,
   type CallMembership,
@@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable";
 
 const logger = rootLogger.getChild("[MatrixLivekitMembers]");
 
-/**
+interface LocalTaggedParticipant {
- * Represents a Matrix call member and their associated LiveKit participation.
+  type: "local";
- * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
+  value$: Behavior<LocalParticipant | null>;
- * or if it has no livekit transport at all.
+}
- */
+interface RemoteTaggedParticipant {
-export interface MatrixLivekitMember {
+  type: "remote";
+  value$: Behavior<RemoteParticipant | null>;
+}
+export type TaggedParticipant =
+  | LocalTaggedParticipant
+  | RemoteTaggedParticipant;
+
+interface MatrixLivekitMember {
   membership$: Behavior<CallMembership>;
-  participant$: Behavior<
-    LocalLivekitParticipant | RemoteLivekitParticipant | null
-  >;
   connection$: Behavior<Connection | null>;
   // participantId: string; We do not want a participantId here since it will be generated by the jwt
   // TODO decide if we can also drop the userId. Its in the matrix membership anyways.
   userId: string;
 }
 
+/**
+ * Represents the local Matrix call member and their associated LiveKit participation.
+ * `livekitParticipant` can be null if the member is not yet connected to the livekit room
+ * or if it has no livekit transport at all.
+ */
+export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
+  participant: LocalTaggedParticipant;
+}
+
+/**
+ * Represents a remote Matrix call member and their associated LiveKit participation.
+ * `livekitParticipant` can be null if the member is not yet connected to the livekit room
+ * or if it has no livekit transport at all.
+ */
+export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
+  participant: RemoteTaggedParticipant;
+}
+
 interface Props {
   scope: ObservableScope;
   membershipsWithTransport$: Behavior<
@@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({
   scope,
   membershipsWithTransport$,
   connectionManager,
-}: Props): { matrixLivekitMembers$: Behavior<Epoch<MatrixLivekitMember[]>> } {
+}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
   /**
    * Stream of all the call members and their associated livekit data (if available).
    */
@@ -110,12 +129,14 @@ export function createMatrixLivekitMembers$({
           logger.debug(
             `Generating member for participantId: ${participantId}, userId: ${userId}`,
           );
+          const { participant$, ...rest } = scope.splitBehavior(data$);
           // will only get called once per `participantId, userId` pair.
           // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
           return {
             participantId,
             userId,
-            ...scope.splitBehavior(data$),
+            participant: { type: "remote" as const, value$: participant$ },
+            ...rest,
           };
         },
       ),

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -29,7 +29,7 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"
 import {
   areLivekitTransportsEqual,
   createMatrixLivekitMembers$,
-  type MatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./MatrixLivekitMembers.ts";
 import { createConnectionManager$ } from "./ConnectionManager.ts";
 import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
@@ -131,8 +131,8 @@ test("bob, carl, then bob joining no tracks yet", () => {
       connectionManager,
     });
 
-    expectObservable(matrixLivekitMembers$).toBe(vMarble, {
+    expectObservable(matrixLivekitItems$).toBe(vMarble, {
-      a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+      a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(1);
         const item = items[0]!;
@@ -147,12 +147,12 @@ test("bob, carl, then bob joining no tracks yet", () => {
             ),
           ),
         });
-        expectObservable(item.participant$).toBe("a", {
+        expectObservable(item.participant.value$).toBe("a", {
           a: null,
         });
         return true;
       }),
-      b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+      b: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(2);
 
@@ -161,7 +161,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
           expectObservable(item.membership$).toBe("a", {
             a: bobMembership,
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
         }
@@ -172,7 +172,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
           expectObservable(item.membership$).toBe("a", {
             a: carlMembership,
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
           expectObservable(item.connection$).toBe("a", {
@@ -189,7 +189,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
         }
         return true;
       }),
-      c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+      c: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(3);
 
@@ -216,7 +216,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
               return true;
             }),
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
         }

src/state/CallViewModelWidget.test.ts

@@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts";
 import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts";
 import { ElementWidgetActions, widget } from "../widget.ts";
 import { E2eeType } from "../e2ee/e2eeType.ts";
+import { MatrixRTCMode } from "../settings/settings.ts";
 
 vi.mock("@livekit/components-core", { spy: true });
 
@@ -34,9 +35,15 @@ vi.mock("../widget", () => ({
   },
 }));
 
-it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
+it.each([
+  [MatrixRTCMode.Legacy],
+  [MatrixRTCMode.Compatibil],
+  [MatrixRTCMode.Matrix_2_0],
+])(
+  "expect leave when ElementWidgetActions.HangupCall is called (%s mode)",
+  async (mode) => {
     const pr = Promise.withResolvers<string>();
-  withCallViewModel(
+    withCallViewModel(mode)(
       {
         remoteParticipants$: constant([aliceParticipant]),
         rtcMembers$: constant([localRtcMember]),
@@ -66,4 +73,5 @@ it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
 
     const source = await pr.promise;
     expect(source).toBe("user");
-});
+  },
+);

src/state/MediaViewModel.test.ts

@@ -20,6 +20,7 @@ import {
   createLocalMedia,
   createRemoteMedia,
   withTestScheduler,
+  mockRemoteParticipant,
 } from "../utils/test";
 import { getValue } from "../utils/observable";
 import { constant } from "./Behavior";
@@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
 
 test("control a participant's volume", () => {
   const setVolumeSpy = vi.fn();
-  const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy });
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({ setVolume: setVolumeSpy }),
+  );
   withTestScheduler(({ expectObservable, schedule }) => {
     schedule("-ab---c---d|", {
       a() {
@@ -88,7 +93,7 @@ test("control a participant's volume", () => {
 });
 
 test("toggle fit/contain for a participant's video", () => {
-  const vm = createRemoteMedia(rtcMembership, {}, {});
+  const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
   withTestScheduler(({ expectObservable, schedule }) => {
     schedule("-ab|", {
       a: () => vm.toggleFitContain(),
@@ -199,3 +204,35 @@ test("switch cameras", async () => {
   });
   expect(deviceId).toBe("front camera");
 });
+
+test("remote media is in waiting state when participant has not yet connected", () => {
+  const vm = createRemoteMedia(rtcMembership, {}, null); // null participant
+  expect(vm.waitingForMedia$.value).toBe(true);
+});
+
+test("remote media is not in waiting state when participant is connected", () => {
+  const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
+  expect(vm.waitingForMedia$.value).toBe(false);
+});
+
+test("remote media is not in waiting state when participant is connected with no publications", () => {
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({
+      getTrackPublication: () => undefined,
+      getTrackPublications: () => [],
+    }),
+  );
+  expect(vm.waitingForMedia$.value).toBe(false);
+});
+
+test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({}),
+    undefined, // No room (no advertised transport)
+  );
+  expect(vm.waitingForMedia$.value).toBe(false);
+});

src/state/MediaViewModel.ts

@@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details.
 
 import {
   type AudioSource,
-  type TrackReferenceOrPlaceholder,
   type VideoSource,
+  type TrackReference,
   observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
@@ -33,7 +33,6 @@ import {
   type Observable,
   Subject,
   combineLatest,
-  distinctUntilKeyChanged,
   filter,
   fromEvent,
   interval,
@@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope";
 export function observeTrackReference$(
   participant: Participant,
   source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
+): Observable<TrackReference | undefined> {
   return observeParticipantMedia(participant).pipe(
-    map(() => ({
+    map(() => participant.getTrackPublication(source)),
-      participant: participant,
+    distinctUntilChanged(),
-      publication: participant.getTrackPublication(source),
+    map((publication) => publication && { participant, publication, source }),
-      source,
-    })),
-    distinctUntilKeyChanged("publication"),
   );
 }
 
@@ -226,7 +222,7 @@ abstract class BaseMediaViewModel {
   /**
    * The LiveKit video track for this media.
    */
-  public readonly video$: Behavior<TrackReferenceOrPlaceholder | null>;
+  public readonly video$: Behavior<TrackReference | undefined>;
   /**
    * Whether there should be a warning that this media is unencrypted.
    */
@@ -241,10 +237,12 @@ abstract class BaseMediaViewModel {
 
   private observeTrackReference$(
     source: Track.Source,
-  ): Behavior<TrackReferenceOrPlaceholder | null> {
+  ): Behavior<TrackReference | undefined> {
     return this.scope.behavior(
       this.participant$.pipe(
-        switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))),
+        switchMap((p) =>
+          !p ? of(undefined) : observeTrackReference$(p, source),
+        ),
       ),
     );
   }
@@ -268,7 +266,7 @@ abstract class BaseMediaViewModel {
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
-    livekitRoom$: Behavior<LivekitRoom | undefined>,
+    protected readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
     public readonly focusUrl$: Behavior<string | undefined>,
     public readonly displayName$: Behavior<string>,
     public readonly mxcAvatarUrl$: Behavior<string | undefined>,
@@ -281,8 +279,8 @@ abstract class BaseMediaViewModel {
         [audio$, this.video$],
         (a, v) =>
           encryptionSystem.kind !== E2eeType.NONE &&
-          (a?.publication?.isEncrypted === false ||
+          (a?.publication.isEncrypted === false ||
-            v?.publication?.isEncrypted === false),
+            v?.publication.isEncrypted === false),
       ),
     );
 
@@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   private readonly videoTrack$: Observable<LocalVideoTrack | null> =
     this.video$.pipe(
       switchMap((v) => {
-        const track = v?.publication?.track;
+        const track = v?.publication.track;
         if (!(track instanceof LocalVideoTrack)) return of(null);
         return merge(
           // Watch for track restarts because they indicate a camera switch.
@@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
  * A remote participant's user media.
  */
 export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
+  /**
+   * Whether we are waiting for this user's LiveKit participant to exist. This
+   * could be because either we or the remote party are still connecting.
+   */
+  public readonly waitingForMedia$ = this.scope.behavior<boolean>(
+    combineLatest(
+      [this.livekitRoom$, this.participant$],
+      (livekitRoom, participant) =>
+        // If livekitRoom is undefined, the user is not attempting to publish on
+        // any transport and so we shouldn't expect a participant. (They might
+        // be a subscribe-only bot for example.)
+        livekitRoom !== undefined && participant === null,
+    ),
+  );
+
   // This private field is used to override the value from the superclass
   private __speaking$: Behavior<boolean>;
   public get speaking$(): Behavior<boolean> {

src/state/UserMedia.ts

@@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions";
 import { observeSpeaker$ } from "./observeSpeaker.ts";
 import { generateItems } from "../utils/observable.ts";
 import { ScreenShare } from "./ScreenShare.ts";
+import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts";
 
 /**
  * Sorting bins defining the order in which media tiles appear in the layout.
@@ -68,12 +69,13 @@ enum SortingBin {
  * for inclusion in the call layout and tracks associated screen shares.
  */
 export class UserMedia {
-  public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
+  public readonly vm: UserMediaViewModel =
+    this.participant.type === "local"
       ? new LocalUserMediaViewModel(
           this.scope,
           this.id,
           this.userId,
-        this.participant$ as Behavior<LocalParticipant | null>,
+          this.participant.value$,
           this.encryptionSystem,
           this.livekitRoom$,
           this.focusUrl$,
@@ -87,7 +89,7 @@ export class UserMedia {
           this.scope,
           this.id,
           this.userId,
-        this.participant$ as Behavior<RemoteParticipant | null>,
+          this.participant.value$,
           this.encryptionSystem,
           this.livekitRoom$,
           this.focusUrl$,
@@ -102,6 +104,11 @@ export class UserMedia {
     observeSpeaker$(this.vm.speaking$),
   );
 
+  // TypeScript needs this widening of the type to happen in a separate statement
+  private readonly participant$: Behavior<
+    LocalParticipant | RemoteParticipant | null
+  > = this.participant.value$;
+
   /**
    * All screen share media associated with this user media.
    */
@@ -184,9 +191,7 @@ export class UserMedia {
     private readonly scope: ObservableScope,
     public readonly id: string,
     private readonly userId: string,
-    private readonly participant$: Behavior<
+    private readonly participant: TaggedParticipant,
-      LocalParticipant | RemoteParticipant | null
-    >,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
     private readonly focusUrl$: Behavior<string | undefined>,

src/tile/GridTile.test.tsx

@@ -12,7 +12,11 @@ import { axe } from "vitest-axe";
 import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
 
 import { GridTile } from "./GridTile";
-import { mockRtcMembership, createRemoteMedia } from "../utils/test";
+import {
+  mockRtcMembership,
+  createRemoteMedia,
+  mockRemoteParticipant,
+} from "../utils/test";
 import { GridTileViewModel } from "../state/TileViewModel";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
@@ -31,11 +35,11 @@ test("GridTile is accessible", async () => {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",
     },
-    {
+    mockRemoteParticipant({
       setVolume() {},
       getTrackPublication: () =>
         ({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
-    },
+    }),
   );
 
   const fakeRtcSession = {

src/tile/GridTile.tsx

@@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps {
   vm: UserMediaViewModel;
   mirror: boolean;
   locallyMuted: boolean;
+  waitingForMedia?: boolean;
   primaryButton?: ReactNode;
   menuStart?: ReactNode;
   menuEnd?: ReactNode;
@@ -79,6 +80,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   vm,
   showSpeakingIndicators,
   locallyMuted,
+  waitingForMedia,
   primaryButton,
   menuStart,
   menuEnd,
@@ -148,7 +150,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   const tile = (
     <MediaView
       ref={ref}
-      video={video ?? undefined}
+      video={video}
       userId={vm.userId}
       unencryptedWarning={unencryptedWarning}
       encryptionStatus={encryptionStatus}
@@ -194,7 +196,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
       raisedHandTime={handRaised ?? undefined}
       currentReaction={reaction ?? undefined}
       raisedHandOnClick={raisedHandOnClick}
-      localParticipant={vm.local}
+      waitingForMedia={waitingForMedia}
       focusUrl={focusUrl}
       audioStreamStats={audioStreamStats}
       videoStreamStats={videoStreamStats}
@@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
   ...props
 }) => {
   const { t } = useTranslation();
+  const waitingForMedia = useBehavior(vm.waitingForMedia$);
   const locallyMuted = useBehavior(vm.locallyMuted$);
   const localVolume = useBehavior(vm.localVolume$);
   const onSelectMute = useCallback(
@@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
     <UserMediaTile
       ref={ref}
       vm={vm}
+      waitingForMedia={waitingForMedia}
       locallyMuted={locallyMuted}
       mirror={false}
       menuStart={

src/tile/MediaView.test.tsx

@@ -47,7 +47,6 @@ describe("MediaView", () => {
     video: trackReference,
     userId: "@alice:example.com",
     mxcAvatarUrl: undefined,
-    localParticipant: false,
     focusable: true,
   };
 
@@ -66,24 +65,13 @@ describe("MediaView", () => {
     });
   });
 
-  describe("with no participant", () => {
+  describe("with no video", () => {
-    it("shows avatar for local user", () => {
+    it("shows avatar", () => {
-      render(
+      render(<MediaView {...baseProps} video={undefined} />);
-        <MediaView {...baseProps} video={undefined} localParticipant={true} />,
-      );
       expect(
         screen.getByRole("img", { name: "@alice:example.com" }),
       ).toBeVisible();
-      expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
+      expect(screen.queryByTestId("video")).toBe(null);
-    });
-    it("shows avatar and label for remote user", () => {
-      render(
-        <MediaView {...baseProps} video={undefined} localParticipant={false} />,
-      );
-      expect(
-        screen.getByRole("img", { name: "@alice:example.com" }),
-      ).toBeVisible();
-      expect(screen.getByText("Waiting for media...")).toBeVisible();
     });
   });
 
@@ -94,6 +82,22 @@ describe("MediaView", () => {
     });
   });
 
+  describe("waitingForMedia", () => {
+    test("defaults to false", () => {
+      render(<MediaView {...baseProps} />);
+      expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
+    });
+    test("shows and is accessible", async () => {
+      const { container } = render(
+        <TooltipProvider>
+          <MediaView {...baseProps} waitingForMedia={true} />
+        </TooltipProvider>,
+      );
+      expect(await axe(container)).toHaveNoViolations();
+      expect(screen.getByText("Waiting for media...")).toBeVisible();
+    });
+  });
+
   describe("unencryptedWarning", () => {
     test("is shown and accessible", async () => {
       const { container } = render(

src/tile/MediaView.tsx

@@ -43,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   raisedHandTime?: Date;
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
-  localParticipant: boolean;
+  waitingForMedia?: boolean;
   audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   // The focus url, mainly for debugging purposes
@@ -71,7 +71,7 @@ export const MediaView: FC<Props> = ({
   raisedHandTime,
   currentReaction,
   raisedHandOnClick,
-  localParticipant,
+  waitingForMedia,
   audioStreamStats,
   videoStreamStats,
   focusUrl,
@@ -129,7 +129,7 @@ export const MediaView: FC<Props> = ({
             />
           )}
         </div>
-        {!video && !localParticipant && (
+        {waitingForMedia && (
           <div className={styles.status}>
             {t("video_tile.waiting_for_media")}
           </div>

src/tile/SpotlightTile.test.tsx

@@ -17,6 +17,7 @@ import {
   mockRtcMembership,
   createLocalMedia,
   createRemoteMedia,
+  mockRemoteParticipant,
 } from "../utils/test";
 import { SpotlightTileViewModel } from "../state/TileViewModel";
 import { constant } from "../state/Behavior";
@@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",
     },
-    {},
+    mockRemoteParticipant({}),
   );
 
   const vm2 = createLocalMedia(

src/tile/SpotlightTile.tsx

@@ -38,6 +38,7 @@ import {
   type MediaViewModel,
   ScreenShareViewModel,
   type UserMediaViewModel,
+  type RemoteUserMediaViewModel,
 } from "../state/MediaViewModel";
 import { useInitial } from "../useInitial";
 import { useMergedRefs } from "../useMergedRefs";
@@ -84,6 +85,21 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
 
 SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";
 
+interface SpotlightRemoteUserMediaItemProps
+  extends SpotlightUserMediaItemBaseProps {
+  vm: RemoteUserMediaViewModel;
+}
+
+const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
+  vm,
+  ...props
+}) => {
+  const waitingForMedia = useBehavior(vm.waitingForMedia$);
+  return (
+    <MediaView waitingForMedia={waitingForMedia} mirror={false} {...props} />
+  );
+};
+
 interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
   vm: UserMediaViewModel;
 }
@@ -103,7 +119,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
   return vm instanceof LocalUserMediaViewModel ? (
     <SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
   ) : (
-    <MediaView mirror={false} {...baseProps} />
+    <SpotlightRemoteUserMediaItem vm={vm} {...baseProps} />
   );
 };
 

src/utils/observable.test.ts

@@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { test } from "vitest";
+import { expect, test } from "vitest";
-import { Subject } from "rxjs";
+import { type Observable, of, Subject, switchMap } from "rxjs";
 
 import { withTestScheduler } from "./test";
-import { generateItems, pauseWhen } from "./observable";
+import { filterBehavior, generateItems, pauseWhen } from "./observable";
+import { type Behavior } from "../state/Behavior";
 
 test("pauseWhen", () => {
   withTestScheduler(({ behavior, expectObservable }) => {
@@ -72,3 +73,31 @@ test("generateItems", () => {
     expectObservable(scope4$).toBe(scope4Marbles);
   });
 });
+
+test("filterBehavior", () => {
+  withTestScheduler(({ behavior, expectObservable }) => {
+    // Filtering the input should segment it into 2 modes of non-null behavior.
+    const inputMarbles = "   abcxabx";
+    const filteredMarbles = "a--xa-x";
+
+    const input$ = behavior(inputMarbles, {
+      a: "a",
+      b: "b",
+      c: "c",
+      x: null,
+    });
+    const filtered$: Observable<Behavior<string> | null> = input$.pipe(
+      filterBehavior((value) => typeof value === "string"),
+    );
+
+    expectObservable(filtered$).toBe(filteredMarbles, {
+      a: expect.any(Object),
+      x: null,
+    });
+    expectObservable(
+      filtered$.pipe(
+        switchMap((value$) => (value$ === null ? of(null) : value$)),
+      ),
+    ).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null });
+  });
+});

src/utils/observable.ts

@@ -22,6 +22,7 @@ import {
   withLatestFrom,
   BehaviorSubject,
   type OperatorFunction,
+  distinctUntilChanged,
 } from "rxjs";
 
 import { type Behavior } from "../state/Behavior";
@@ -185,6 +186,28 @@ export function generateItemsWithEpoch<
   );
 }
 
+/**
+ * Segments a behavior into periods during which its value matches the filter
+ * (outputting a behavior with a narrowed type) and periods during which it does
+ * not match (outputting null).
+ */
+export function filterBehavior<T, S extends T>(
+  predicate: (value: T) => value is S,
+): OperatorFunction<T, Behavior<S> | null> {
+  return (input$) =>
+    input$.pipe(
+      scan<T, BehaviorSubject<S> | null>((acc$, input) => {
+        if (predicate(input)) {
+          const output$ = acc$ ?? new BehaviorSubject(input);
+          output$.next(input);
+          return output$;
+        }
+        return null;
+      }, null),
+      distinctUntilChanged(),
+    );
+}
+
 function generateItemsInternal<
   Input,
   Keys extends [unknown, ...unknown[]],

src/utils/test-viewmodel.ts

@@ -37,6 +37,7 @@ import {
 import { aliceRtcMember, localRtcMember } from "./test-fixtures";
 import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
 import { constant } from "../state/Behavior";
+import { MatrixRTCMode } from "../settings/settings";
 
 mockConfig({ livekit: { livekit_service_url: "https://example.com" } });
 
@@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment(
           setE2EEEnabled: async () => Promise.resolve(),
         }),
       connectionState$: constant(ConnectionState.Connected),
+      matrixRTCMode$: constant(MatrixRTCMode.Legacy),
       ...callViewModelOptions,
     },
     handRaisedSubject$,

src/utils/test.ts

@@ -311,6 +311,8 @@ export function mockLocalParticipant(
     publishTrack: vi.fn(),
     unpublishTracks: vi.fn().mockResolvedValue([]),
     createTracks: vi.fn(),
+    setMicrophoneEnabled: vi.fn(),
+    setCameraEnabled: vi.fn(),
     getTrackPublication: () =>
       ({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
     ...mockEmitter(),
@@ -319,12 +321,12 @@ export function mockLocalParticipant(
 }
 
 export function createLocalMedia(
-  localRtcMember: CallMembership,
+  rtcMember: CallMembership,
   roomMember: Partial<RoomMember>,
   localParticipant: LocalParticipant,
   mediaDevices: MediaDevices,
 ): LocalUserMediaViewModel {
-  const member = mockMatrixRoomMember(localRtcMember, roomMember);
+  const member = mockMatrixRoomMember(rtcMember, roomMember);
   return new LocalUserMediaViewModel(
     testScope(),
     "local",
@@ -359,23 +361,26 @@ export function mockRemoteParticipant(
 }
 
 export function createRemoteMedia(
-  localRtcMember: CallMembership,
+  rtcMember: CallMembership,
   roomMember: Partial<RoomMember>,
-  participant: Partial<RemoteParticipant>,
+  participant: RemoteParticipant | null,
+  livekitRoom: LivekitRoom | undefined = mockLivekitRoom(
+    {},
+    {
+      remoteParticipants$: of(participant ? [participant] : []),
+    },
+  ),
 ): RemoteUserMediaViewModel {
-  const member = mockMatrixRoomMember(localRtcMember, roomMember);
+  const member = mockMatrixRoomMember(rtcMember, roomMember);
-  const remoteParticipant = mockRemoteParticipant(participant);
   return new RemoteUserMediaViewModel(
     testScope(),
     "remote",
     member.userId,
-    of(remoteParticipant),
+    constant(participant),
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
-    constant(
+    constant(livekitRoom),
-      mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
-    ),
     constant("https://rtc-example.org"),
     constant(false),
     constant(member.rawDisplayName ?? "nodisplayname"),

yarn.lock

@@ -3380,14 +3380,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@playwright/test@npm:^1.56.1":
+"@playwright/test@npm:^1.57.0":
-  version: 1.56.1
+  version: 1.57.0
-  resolution: "@playwright/test@npm:1.56.1"
+  resolution: "@playwright/test@npm:1.57.0"
   dependencies:
-    playwright: "npm:1.56.1"
+    playwright: "npm:1.57.0"
   bin:
     playwright: cli.js
-  checksum: 10c0/2b5b0e1f2e6a18f6e5ce6897c7440ca78f64e0b004834e9808e93ad2b78b96366b562ae4366602669cf8ad793a43d85481b58541e74be71e905e732d833dd691
+  checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774
   languageName: node
   linkType: hard
 
@@ -7786,7 +7786,7 @@ __metadata:
     "@opentelemetry/sdk-trace-base": "npm:^2.0.0"
     "@opentelemetry/sdk-trace-web": "npm:^2.0.0"
     "@opentelemetry/semantic-conventions": "npm:^1.25.1"
-    "@playwright/test": "npm:^1.56.1"
+    "@playwright/test": "npm:^1.57.0"
     "@radix-ui/react-dialog": "npm:^1.0.4"
     "@radix-ui/react-slider": "npm:^1.1.2"
     "@radix-ui/react-visually-hidden": "npm:^1.0.3"
@@ -10781,8 +10781,8 @@ __metadata:
   linkType: hard
 
 "matrix-js-sdk@npm:^39.2.0":
-  version: 39.2.0
+  version: 39.3.0
-  resolution: "matrix-js-sdk@npm:39.2.0"
+  resolution: "matrix-js-sdk@npm:39.3.0"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0"
@@ -10798,7 +10798,7 @@ __metadata:
     sdp-transform: "npm:^3.0.0"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:13"
-  checksum: 10c0/f8b5261de2744305330ba3952821ca9303698170bfd3a0ff8a767b9286d4e8d4ed5aaf6fbaf8a1e8ff9dbd859102a2a47d882787e2da3b3078965bec00157959
+  checksum: 10c0/031c9ec042e00c32dc531f82fc59c64cc25fb665abfc642b1f0765c530d60684f8bd63daf0cdd0dbe96b4f87ea3f4148f9d3f024a59d57eceaec1ce5d0164755
   languageName: node
   linkType: hard
 
@@ -11735,36 +11735,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pkg-dir@npm:^5.0.0":
+"playwright-core@npm:1.57.0":
-  version: 5.0.0
+  version: 1.57.0
-  resolution: "pkg-dir@npm:5.0.0"
+  resolution: "playwright-core@npm:1.57.0"
-  dependencies:
-    find-up: "npm:^5.0.0"
-  checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5
-  languageName: node
-  linkType: hard
-
-"playwright-core@npm:1.56.1":
-  version: 1.56.1
-  resolution: "playwright-core@npm:1.56.1"
   bin:
     playwright-core: cli.js
-  checksum: 10c0/ffd40142b99c68678b387445d5b42f1fee4ab0b65d983058c37f342e5629f9cdbdac0506ea80a0dfd41a8f9f13345bad54e9a8c35826ef66dc765f4eb3db8da7
+  checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9
   languageName: node
   linkType: hard
 
-"playwright@npm:1.56.1":
+"playwright@npm:1.57.0":
-  version: 1.56.1
+  version: 1.57.0
-  resolution: "playwright@npm:1.56.1"
+  resolution: "playwright@npm:1.57.0"
   dependencies:
     fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.56.1"
+    playwright-core: "npm:1.57.0"
   dependenciesMeta:
     fsevents:
       optional: true
   bin:
     playwright: cli.js
-  checksum: 10c0/8e9965aede86df0f4722063385748498977b219630a40a10d1b82b8bd8d4d4e9b6b65ecbfa024331a30800163161aca292fb6dd7446c531a1ad25f4155625ab4
+  checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899
   languageName: node
   linkType: hard