temp

src/state/remoteMembers/Connection.test.ts

@@ -0,0 +1,703 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  afterEach,
+  describe,
+  expect,
+  it,
+  type Mock,
+  type MockedObject,
+  onTestFinished,
+  vi,
+} from "vitest";
+import { BehaviorSubject, of } from "rxjs";
+import {
+  ConnectionState,
+  type LocalParticipant,
+  type RemoteParticipant,
+  type Room as LivekitRoom,
+  RoomEvent,
+  type RoomOptions,
+} from "livekit-client";
+import fetchMock from "fetch-mock";
+import EventEmitter from "events";
+import { type IOpenIDToken } from "matrix-js-sdk";
+
+import type {
+  CallMembership,
+  LivekitTransport,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  type ConnectionOpts,
+  type ConnectionState,
+  type PublishingParticipant,
+  RemoteConnection,
+} from "./Connection.ts";
+import { ObservableScope } from "../ObservableScope.ts";
+import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
+import { FailToGetOpenIdToken } from "../../utils/errors.ts";
+import { PublishConnection } from "../ownMember/PublishConnection.ts";
+import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts";
+import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
+import { type MuteStates } from "../MuteStates.ts";
+
+let testScope: ObservableScope;
+
+let client: MockedObject<OpenIDClientParts>;
+
+let fakeLivekitRoom: MockedObject<LivekitRoom>;
+
+let localParticipantEventEmiter: EventEmitter;
+let fakeLocalParticipant: MockedObject<LocalParticipant>;
+
+let fakeRoomEventEmiter: EventEmitter;
+let fakeMembershipsFocusMap$: BehaviorSubject<
+  { membership: CallMembership; transport: LivekitTransport }[]
+>;
+
+const livekitFocus: LivekitTransport = {
+  livekit_alias: "!roomID:example.org",
+  livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
+  type: "livekit",
+};
+
+function setupTest(): void {
+  testScope = new ObservableScope();
+  client = vi.mocked<OpenIDClientParts>({
+    getOpenIdToken: vi.fn().mockResolvedValue({
+      access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
+      token_type: "Bearer",
+      matrix_server_name: "example.org",
+      expires_in: 3600,
+    }),
+    getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
+  } as unknown as OpenIDClientParts);
+  fakeMembershipsFocusMap$ = new BehaviorSubject<
+    { membership: CallMembership; transport: LivekitTransport }[]
+  >([]);
+
+  localParticipantEventEmiter = new EventEmitter();
+
+  fakeLocalParticipant = vi.mocked<LocalParticipant>({
+    identity: "@me:example.org",
+    isMicrophoneEnabled: vi.fn().mockReturnValue(true),
+    getTrackPublication: vi.fn().mockReturnValue(undefined),
+    on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter),
+    off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter),
+    addListener: localParticipantEventEmiter.addListener.bind(
+      localParticipantEventEmiter,
+    ),
+    removeListener: localParticipantEventEmiter.removeListener.bind(
+      localParticipantEventEmiter,
+    ),
+    removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(
+      localParticipantEventEmiter,
+    ),
+  } as unknown as LocalParticipant);
+  fakeRoomEventEmiter = new EventEmitter();
+
+  fakeLivekitRoom = vi.mocked<LivekitRoom>({
+    connect: vi.fn(),
+    disconnect: vi.fn(),
+    remoteParticipants: new Map(),
+    localParticipant: fakeLocalParticipant,
+    state: ConnectionState.Disconnected,
+    on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
+    off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
+    addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
+    removeListener:
+      fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
+    removeAllListeners:
+      fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
+    setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
+  } as unknown as LivekitRoom);
+}
+
+function setupRemoteConnection(): RemoteConnection {
+  const opts: ConnectionOpts = {
+    client: client,
+    transport: livekitFocus,
+    remoteTransports$: fakeMembershipsFocusMap$,
+    scope: testScope,
+    livekitRoomFactory: () => fakeLivekitRoom,
+  };
+
+  fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
+    return {
+      status: 200,
+      body: {
+        url: "wss://matrix-rtc.m.localhost/livekit/sfu",
+        jwt: "ATOKEN",
+      },
+    };
+  });
+
+  fakeLivekitRoom.connect.mockResolvedValue(undefined);
+
+  return new RemoteConnection(opts, undefined);
+}
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.clearAllMocks();
+  fetchMock.reset();
+});
+
+describe("Start connection states", () => {
+  it("start in initialized state", () => {
+    setupTest();
+
+    const opts: ConnectionOpts = {
+      client: client,
+      transport: livekitFocus,
+      remoteTransports$: fakeMembershipsFocusMap$,
+      scope: testScope,
+      livekitRoomFactory: () => fakeLivekitRoom,
+    };
+    const connection = new RemoteConnection(opts, undefined);
+
+    expect(connection.state$.getValue().state).toEqual("Initialized");
+  });
+
+  it("fail to getOpenId token then error state", async () => {
+    setupTest();
+    vi.useFakeTimers();
+
+    const opts: ConnectionOpts = {
+      client: client,
+      transport: livekitFocus,
+      remoteTransports$: fakeMembershipsFocusMap$,
+      scope: testScope,
+      livekitRoomFactory: () => fakeLivekitRoom,
+    };
+
+    const connection = new RemoteConnection(opts, undefined);
+
+    const capturedStates: ConnectionState[] = [];
+    const s = connection.state$.subscribe((value) => {
+      capturedStates.push(value);
+    });
+    onTestFinished(() => s.unsubscribe());
+
+    const deferred = Promise.withResolvers<IOpenIDToken>();
+
+    client.getOpenIdToken.mockImplementation(
+      async (): Promise<IOpenIDToken> => {
+        return await deferred.promise;
+      },
+    );
+
+    connection.start().catch(() => {
+      // expected to throw
+    });
+
+    let capturedState = capturedStates.pop();
+    expect(capturedState).toBeDefined();
+    expect(capturedState!.state).toEqual("FetchingConfig");
+
+    deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
+
+    await vi.runAllTimersAsync();
+
+    capturedState = capturedStates.pop();
+    if (capturedState!.state === "FailedToStart") {
+      expect(capturedState!.error.message).toEqual("Something went wrong");
+      expect(capturedState!.transport.livekit_alias).toEqual(
+        livekitFocus.livekit_alias,
+      );
+    } else {
+      expect.fail(
+        "Expected FailedToStart state but got " + capturedState?.state,
+      );
+    }
+  });
+
+  it("fail to get JWT token and error state", async () => {
+    setupTest();
+    vi.useFakeTimers();
+
+    const opts: ConnectionOpts = {
+      client: client,
+      transport: livekitFocus,
+      remoteTransports$: fakeMembershipsFocusMap$,
+      scope: testScope,
+      livekitRoomFactory: () => fakeLivekitRoom,
+    };
+
+    const connection = new RemoteConnection(opts, undefined);
+
+    const capturedStates: ConnectionState[] = [];
+    const s = connection.state$.subscribe((value) => {
+      capturedStates.push(value);
+    });
+    onTestFinished(() => s.unsubscribe());
+
+    const deferredSFU = Promise.withResolvers<void>();
+    // mock the /sfu/get call
+    fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => {
+      await deferredSFU.promise;
+      return {
+        status: 500,
+        body: "Internal Server Error",
+      };
+    });
+
+    connection.start().catch(() => {
+      // expected to throw
+    });
+
+    let capturedState = capturedStates.pop();
+    expect(capturedState).toBeDefined();
+    expect(capturedState?.state).toEqual("FetchingConfig");
+
+    deferredSFU.resolve();
+    await vi.runAllTimersAsync();
+
+    capturedState = capturedStates.pop();
+
+    if (capturedState?.state === "FailedToStart") {
+      expect(capturedState?.error.message).toContain(
+        "SFU Config fetch failed with exception Error",
+      );
+      expect(capturedState?.transport.livekit_alias).toEqual(
+        livekitFocus.livekit_alias,
+      );
+    } else {
+      expect.fail(
+        "Expected FailedToStart state but got " + capturedState?.state,
+      );
+    }
+  });
+
+  it("fail to connect to livekit error state", async () => {
+    setupTest();
+    vi.useFakeTimers();
+
+    const opts: ConnectionOpts = {
+      client: client,
+      transport: livekitFocus,
+      remoteTransports$: fakeMembershipsFocusMap$,
+      scope: testScope,
+      livekitRoomFactory: () => fakeLivekitRoom,
+    };
+
+    const connection = new RemoteConnection(opts, undefined);
+
+    const capturedStates: ConnectionState[] = [];
+    const s = connection.state$.subscribe((value) => {
+      capturedStates.push(value);
+    });
+    onTestFinished(() => s.unsubscribe());
+
+    const deferredSFU = Promise.withResolvers<void>();
+    // mock the /sfu/get call
+    fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
+      return {
+        status: 200,
+        body: {
+          url: "wss://matrix-rtc.m.localhost/livekit/sfu",
+          jwt: "ATOKEN",
+        },
+      };
+    });
+
+    fakeLivekitRoom.connect.mockImplementation(async () => {
+      await deferredSFU.promise;
+      throw new Error("Failed to connect to livekit");
+    });
+
+    connection.start().catch(() => {
+      // expected to throw
+    });
+
+    let capturedState = capturedStates.pop();
+    expect(capturedState).toBeDefined();
+
+    expect(capturedState?.state).toEqual("FetchingConfig");
+
+    deferredSFU.resolve();
+    await vi.runAllTimersAsync();
+
+    capturedState = capturedStates.pop();
+
+    if (capturedState && capturedState?.state === "FailedToStart") {
+      expect(capturedState.error.message).toContain(
+        "Failed to connect to livekit",
+      );
+      expect(capturedState.transport.livekit_alias).toEqual(
+        livekitFocus.livekit_alias,
+      );
+    } else {
+      expect.fail(
+        "Expected FailedToStart state but got " + JSON.stringify(capturedState),
+      );
+    }
+  });
+
+  it("connection states happy path", async () => {
+    vi.useFakeTimers();
+    setupTest();
+
+    const connection = setupRemoteConnection();
+
+    const capturedStates: ConnectionState[] = [];
+    const s = connection.state$.subscribe((value) => {
+      capturedStates.push(value);
+    });
+    onTestFinished(() => s.unsubscribe());
+
+    await connection.start();
+    await vi.runAllTimersAsync();
+
+    const initialState = capturedStates.shift();
+    expect(initialState?.state).toEqual("Initialized");
+    const fetchingState = capturedStates.shift();
+    expect(fetchingState?.state).toEqual("FetchingConfig");
+    const connectingState = capturedStates.shift();
+    expect(connectingState?.state).toEqual("ConnectingToLkRoom");
+    const connectedState = capturedStates.shift();
+    expect(connectedState?.state).toEqual("ConnectedToLkRoom");
+  });
+
+  it("shutting down the scope should stop the connection", async () => {
+    setupTest();
+    vi.useFakeTimers();
+
+    const connection = setupRemoteConnection();
+    await connection.start();
+
+    const stopSpy = vi.spyOn(connection, "stop");
+    testScope.end();
+
+    expect(stopSpy).toHaveBeenCalled();
+    expect(fakeLivekitRoom.disconnect).toHaveBeenCalled();
+  });
+});
+
+function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
+  return {
+    identity: id,
+  } as unknown as RemoteParticipant;
+}
+
+function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
+  return {
+    userId,
+    deviceId,
+  } as unknown as CallMembership;
+}
+
+describe("Publishing participants observations", () => {
+  it("should emit the list of publishing participants", async () => {
+    setupTest();
+
+    const connection = setupRemoteConnection();
+
+    const bobIsAPublisher = Promise.withResolvers<void>();
+    const danIsAPublisher = Promise.withResolvers<void>();
+    const observedPublishers: PublishingParticipant[][] = [];
+    const s = connection.allLivekitParticipants$.subscribe((publishers) => {
+      observedPublishers.push(publishers);
+      if (
+        publishers.some(
+          (p) => p.participant?.identity === "@bob:example.org:DEV111",
+        )
+      ) {
+        bobIsAPublisher.resolve();
+      }
+      if (
+        publishers.some(
+          (p) => p.participant?.identity === "@dan:example.org:DEV333",
+        )
+      ) {
+        danIsAPublisher.resolve();
+      }
+    });
+    onTestFinished(() => s.unsubscribe());
+    // The publishingParticipants$ observable is derived from the current members of the
+    // livekitRoom and the rtc membership in order to publish the members that are publishing
+    // on this connection.
+
+    let participants: RemoteParticipant[] = [
+      fakeRemoteLivekitParticipant("@alice:example.org:DEV000"),
+      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+      fakeRemoteLivekitParticipant("@carol:example.org:DEV222"),
+      fakeRemoteLivekitParticipant("@dan:example.org:DEV333"),
+    ];
+
+    // Let's simulate 3 members on the livekitRoom
+    vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
+      new Map(participants.map((p) => [p.identity, p])),
+    );
+
+    for (const participant of participants) {
+      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
+    }
+
+    // At this point there should be no publishers
+    expect(observedPublishers.pop()!.length).toEqual(0);
+
+    const otherFocus: LivekitTransport = {
+      livekit_alias: "!roomID:example.org",
+      livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt",
+      type: "livekit",
+    };
+
+    const rtcMemberships = [
+      // Say bob is on the same focus
+      {
+        membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
+        transport: livekitFocus,
+      },
+      // Alice and carol is on a different focus
+      {
+        membership: fakeRtcMemberShip("@alice:example.org", "DEV000"),
+        transport: otherFocus,
+      },
+      {
+        membership: fakeRtcMemberShip("@carol:example.org", "DEV222"),
+        transport: otherFocus,
+      },
+      // NO DAVE YET
+    ];
+    // signal this change in rtc memberships
+    fakeMembershipsFocusMap$.next(rtcMemberships);
+
+    // We should have bob has a publisher now
+    await bobIsAPublisher.promise;
+    const publishers = observedPublishers.pop();
+    expect(publishers?.length).toEqual(1);
+    expect(publishers?.[0].participant?.identity).toEqual(
+      "@bob:example.org:DEV111",
+    );
+
+    // Now let's make dan join the rtc memberships
+    rtcMemberships.push({
+      membership: fakeRtcMemberShip("@dan:example.org", "DEV333"),
+      transport: livekitFocus,
+    });
+    fakeMembershipsFocusMap$.next(rtcMemberships);
+
+    // We should have bob and dan has publishers now
+    await danIsAPublisher.promise;
+    const twoPublishers = observedPublishers.pop();
+    expect(twoPublishers?.length).toEqual(2);
+    expect(
+      twoPublishers?.some(
+        (p) => p.participant?.identity === "@bob:example.org:DEV111",
+      ),
+    ).toBeTruthy();
+    expect(
+      twoPublishers?.some(
+        (p) => p.participant?.identity === "@dan:example.org:DEV333",
+      ),
+    ).toBeTruthy();
+
+    // Now let's make bob leave the livekit room
+    participants = participants.filter(
+      (p) => p.identity !== "@bob:example.org:DEV111",
+    );
+    vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
+      new Map(participants.map((p) => [p.identity, p])),
+    );
+    fakeRoomEventEmiter.emit(
+      RoomEvent.ParticipantDisconnected,
+      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+    );
+
+    const updatedPublishers = observedPublishers.pop();
+    // Bob is not connected to the room but he is still in the rtc memberships declaring that
+    // he is using that focus to publish, so he should still appear as a publisher
+    expect(updatedPublishers?.length).toEqual(2);
+    const pp = updatedPublishers?.find(
+      (p) => p.membership.userId == "@bob:example.org",
+    );
+    expect(pp).toBeDefined();
+    expect(pp!.participant).not.toBeDefined();
+    expect(
+      updatedPublishers?.some(
+        (p) => p.participant?.identity === "@dan:example.org:DEV333",
+      ),
+    ).toBeTruthy();
+    // Now if bob is not in the rtc memberships, he should disappear
+    const noBob = rtcMemberships.filter(
+      ({ membership }) => membership.userId !== "@bob:example.org",
+    );
+    fakeMembershipsFocusMap$.next(noBob);
+    expect(observedPublishers.pop()?.length).toEqual(1);
+  });
+
+  it("should be scoped to parent scope", (): void => {
+    setupTest();
+
+    const connection = setupRemoteConnection();
+
+    let observedPublishers: PublishingParticipant[][] = [];
+    const s = connection.allLivekitParticipants$.subscribe((publishers) => {
+      observedPublishers.push(publishers);
+    });
+    onTestFinished(() => s.unsubscribe());
+
+    let participants: RemoteParticipant[] = [
+      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+    ];
+
+    // Let's simulate 3 members on the livekitRoom
+    vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
+      new Map(participants.map((p) => [p.identity, p])),
+    );
+
+    for (const participant of participants) {
+      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
+    }
+
+    // At this point there should be no publishers
+    expect(observedPublishers.pop()!.length).toEqual(0);
+
+    const rtcMemberships = [
+      // Say bob is on the same focus
+      {
+        membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
+        transport: livekitFocus,
+      },
+    ];
+    // signal this change in rtc memberships
+    fakeMembershipsFocusMap$.next(rtcMemberships);
+
+    // We should have bob has a publisher now
+    const publishers = observedPublishers.pop();
+    expect(publishers?.length).toEqual(1);
+    expect(publishers?.[0].participant?.identity).toEqual(
+      "@bob:example.org:DEV111",
+    );
+
+    // end the parent scope
+    testScope.end();
+    observedPublishers = [];
+
+    // SHOULD NOT emit any more publishers as the scope is ended
+    participants = participants.filter(
+      (p) => p.identity !== "@bob:example.org:DEV111",
+    );
+    vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
+      new Map(participants.map((p) => [p.identity, p])),
+    );
+    fakeRoomEventEmiter.emit(
+      RoomEvent.ParticipantDisconnected,
+      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+    );
+
+    expect(observedPublishers.length).toEqual(0);
+  });
+});
+
+describe("PublishConnection", () => {
+  // let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
+  let roomFactoryMock: Mock<() => LivekitRoom>;
+  let muteStates: MockedObject<MuteStates>;
+
+  function setUpPublishConnection(): void {
+    setupTest();
+
+    roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
+
+    muteStates = mockMuteStates();
+
+    // fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
+    //   name: "BackgroundBlur",
+    //   restart: vi.fn().mockResolvedValue(undefined),
+    //   setOptions: vi.fn().mockResolvedValue(undefined),
+    //   getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
+    //   isRunning: vi.fn().mockReturnValue(false)
+    // });
+  }
+
+  describe("Livekit room creation", () => {
+    function createSetup(): void {
+      setUpPublishConnection();
+
+      const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
+        supported: true,
+        processor: undefined,
+      });
+
+      const opts: ConnectionOpts = {
+        client: client,
+        transport: livekitFocus,
+        remoteTransports$: fakeMembershipsFocusMap$,
+        scope: testScope,
+        livekitRoomFactory: roomFactoryMock,
+      };
+
+      const audioInput = {
+        available$: of(new Map([["mic1", { id: "mic1" }]])),
+        selected$: new BehaviorSubject({ id: "mic1" }),
+        select(): void {},
+      };
+
+      const videoInput = {
+        available$: of(new Map([["cam1", { id: "cam1" }]])),
+        selected$: new BehaviorSubject({ id: "cam1" }),
+        select(): void {},
+      };
+
+      const audioOutput = {
+        available$: of(new Map([["speaker", { id: "speaker" }]])),
+        selected$: new BehaviorSubject({ id: "speaker" }),
+        select(): void {},
+      };
+
+      // TODO understand what is wrong with our mocking that requires ts-expect-error
+      const fakeDevices = mockMediaDevices({
+        // @ts-expect-error Mocking only
+        audioInput,
+        // @ts-expect-error Mocking only
+        videoInput,
+        // @ts-expect-error Mocking only
+        audioOutput,
+      });
+
+      new PublishConnection(
+        opts,
+        fakeDevices,
+        muteStates,
+        undefined,
+        fakeTrackProcessorSubject$,
+      );
+    }
+
+    it("should create room with proper initial audio and video settings", () => {
+      createSetup();
+
+      expect(roomFactoryMock).toHaveBeenCalled();
+
+      const lastCallArgs =
+        roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
+
+      const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
+      expect(roomOptions).toBeDefined();
+
+      expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
+      expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
+      expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
+    });
+
+    it("respect controlledAudioDevices", () => {
+      // TODO: Refactor the code to make it testable.
+      // The UrlParams module is a singleton has a cache and is very hard to test.
+      // This breaks other tests as well if not handled properly.
+      // vi.mock(import("./../UrlParams"), () => {
+      //   return {
+      //     getUrlParams: vi.fn().mockReturnValue({
+      //       controlledAudioDevices: true
+      //     })
+      //   };
+      // });
+    });
+  });
+});

src/state/remoteMembers/Connection.ts

@@ -0,0 +1,290 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  connectedParticipantsObserver,
+  connectionStateObserver,
+} from "@livekit/components-core";
+import {
+  ConnectionError,
+  type ConnectionState as LivekitConenctionState,
+  type E2EEOptions,
+  type RemoteParticipant,
+  Room as LivekitRoom,
+  type RoomOptions,
+} from "livekit-client";
+import {
+  type CallMembership,
+  type LivekitTransport,
+} from "matrix-js-sdk/lib/matrixrtc";
+import { BehaviorSubject, combineLatest, type Observable } from "rxjs";
+import { type Logger } from "matrix-js-sdk/lib/logger";
+
+import {
+  getSFUConfigWithOpenID,
+  type OpenIDClientParts,
+  type SFUConfig,
+} from "../../livekit/openIDSFU.ts";
+import { type Behavior } from "../Behavior.ts";
+import { type ObservableScope } from "../ObservableScope.ts";
+import { defaultLiveKitOptions } from "../../livekit/options.ts";
+import {
+  InsufficientCapacityError,
+  SFURoomCreationRestrictedError,
+} from "../../utils/errors.ts";
+
+export interface ConnectionOpts {
+  /** The media transport to connect to. */
+  transport: LivekitTransport;
+  /** The Matrix client to use for OpenID and SFU config requests. */
+  client: OpenIDClientParts;
+  /** The observable scope to use for this connection. */
+  scope: ObservableScope;
+  /**
+   * An observable of the current RTC call memberships and their associated transports.
+   * Used to differentiate between publishing and subscribging participants on each connection.
+   * Used to find out which rtc member should upload to this connection (publishingParticipants$).
+   * The livekit room gives access to all the users subscribing to this connection, we need
+   * to filter out the ones that are uploading to this connection.
+   */
+  membershipsWithTransport$: Behavior<
+    { membership: CallMembership; transport: LivekitTransport }[]
+  >;
+
+  /** Optional factory to create the LiveKit room, mainly for testing purposes. */
+  livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
+}
+
+export type ConnectionState =
+  | { state: "Initialized" }
+  | { state: "FetchingConfig"; transport: LivekitTransport }
+  | { state: "ConnectingToLkRoom"; transport: LivekitTransport }
+  | { state: "PublishingTracks"; transport: LivekitTransport }
+  | { state: "FailedToStart"; error: Error; transport: LivekitTransport }
+  | {
+      state: "ConnectedToLkRoom";
+      livekitConnectionState$: Observable<LivekitConenctionState>;
+      transport: LivekitTransport;
+    }
+  | { state: "Stopped"; transport: LivekitTransport };
+
+/**
+ * Represents participant publishing or expected to publish on the connection.
+ * It is paired with its associated rtc membership.
+ */
+export type PublishingParticipant = {
+  /**
+   * The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room.
+   */
+  participant: RemoteParticipant | undefined;
+  /**
+   * The rtc call membership associated with this participant.
+   */
+  membership: CallMembership;
+};
+
+/**
+ * A connection to a Matrix RTC LiveKit backend.
+ *
+ * Expose observables for participants and connection state.
+ */
+export class Connection {
+  // Private Behavior
+  private readonly _state$ = new BehaviorSubject<ConnectionState>({
+    state: "Initialized",
+  });
+
+  /**
+   * The current state of the connection to the media transport.
+   */
+  public readonly state$: Behavior<ConnectionState> = this._state$;
+
+  /**
+   * Whether the connection has been stopped.
+   * @see Connection.stop
+   * */
+  protected stopped = false;
+
+  /**
+   * Starts the connection.
+   *
+   * This will:
+   * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
+   * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
+   * 3. Connect to the configured LiveKit room.
+   *
+   * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
+   * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
+   */
+  public async start(): Promise<void> {
+    this.stopped = false;
+    try {
+      this._state$.next({
+        state: "FetchingConfig",
+        transport: this.transport,
+      });
+      const { url, jwt } = await this.getSFUConfigWithOpenID();
+      // If we were stopped while fetching the config, don't proceed to connect
+      if (this.stopped) return;
+
+      this._state$.next({
+        state: "ConnectingToLkRoom",
+        transport: this.transport,
+      });
+      try {
+        await this.livekitRoom.connect(url, jwt);
+      } catch (e) {
+        // LiveKit uses 503 to indicate that the server has hit its track limits.
+        // https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
+        // It also errors with a status code of 200 (yes, really) for room
+        // participant limits.
+        // LiveKit Cloud uses 429 for connection limits.
+        // Either way, all these errors can be explained as "insufficient capacity".
+        if (e instanceof ConnectionError) {
+          if (e.status === 503 || e.status === 200 || e.status === 429) {
+            throw new InsufficientCapacityError();
+          }
+          if (e.status === 404) {
+            // error msg is "Could not establish signal connection: requested room does not exist"
+            // The room does not exist. There are two different modes of operation for the SFU:
+            // - the room is created on the fly when connecting (livekit `auto_create` option)
+            // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
+            // In the first case there will not be a 404, so we are in the second case.
+            throw new SFURoomCreationRestrictedError();
+          }
+        }
+        throw e;
+      }
+      // If we were stopped while connecting, don't proceed to update state.
+      if (this.stopped) return;
+
+      this._state$.next({
+        state: "ConnectedToLkRoom",
+        transport: this.transport,
+        livekitConnectionState$: connectionStateObserver(this.livekitRoom),
+      });
+    } catch (error) {
+      this._state$.next({
+        state: "FailedToStart",
+        error: error instanceof Error ? error : new Error(`${error}`),
+        transport: this.transport,
+      });
+      throw error;
+    }
+  }
+
+  protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
+    return await getSFUConfigWithOpenID(
+      this.client,
+      this.transport.livekit_service_url,
+      this.transport.livekit_alias,
+    );
+  }
+  /**
+   * Stops the connection.
+   *
+   * This will disconnect from the LiveKit room.
+   * If the connection is already stopped, this is a no-op.
+   */
+  public async stop(): Promise<void> {
+    if (this.stopped) return;
+    await this.livekitRoom.disconnect();
+    this._state$.next({
+      state: "Stopped",
+      transport: this.transport,
+    });
+    this.stopped = true;
+  }
+
+  /**
+   * An observable of the participants that are publishing on this connection.
+   * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
+   * It filters the participants to only those that are associated with a membership that claims to publish on this connection.
+   */
+  public readonly publishingParticipants$: Behavior<PublishingParticipant[]>;
+
+  /**
+   * The media transport to connect to.
+   */
+  public readonly transport: LivekitTransport;
+
+  private readonly client: OpenIDClientParts;
+  /**
+   * Creates a new connection to a matrix RTC LiveKit backend.
+   *
+   * @param livekitRoom - LiveKit room instance to use.
+   * @param opts - Connection options {@link ConnectionOpts}.
+   *
+   */
+  protected constructor(
+    public readonly livekitRoom: LivekitRoom,
+    opts: ConnectionOpts,
+    logger?: Logger,
+  ) {
+    logger?.info(
+      `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
+    );
+    const { transport, client, scope, membershipsWithTransport$ } = opts;
+
+    this.transport = transport;
+    this.client = client;
+
+    const participantsIncludingSubscribers$: Behavior<RemoteParticipant[]> =
+      scope.behavior(connectedParticipantsObserver(this.livekitRoom), []);
+
+    this.publishingParticipants$ = scope.behavior(
+      combineLatest(
+        [participantsIncludingSubscribers$, membershipsWithTransport$],
+        (participants, remoteTransports) =>
+          remoteTransports
+            // Find all members that claim to publish on this connection
+            .flatMap(({ membership, transport }) =>
+              transport.livekit_service_url ===
+              this.transport.livekit_service_url
+                ? [membership]
+                : [],
+            )
+            // Pair with their associated LiveKit participant (if any)
+            .map((membership) => {
+              const id = `${membership.userId}:${membership.deviceId}`;
+              const participant = participants.find((p) => p.identity === id);
+              return { participant, membership };
+            }),
+      ),
+      [],
+    );
+
+    scope.onEnd(() => void this.stop());
+  }
+}
+
+/**
+ * A remote connection to the Matrix RTC LiveKit backend.
+ *
+ * This connection is used for subscribing to remote participants.
+ * It does not publish any local tracks.
+ */
+export class RemoteConnection extends Connection {
+  /**
+   * Creates a new remote connection to a matrix RTC LiveKit backend.
+   * @param opts
+   * @param sharedE2eeOption - The shared E2EE options to use for the connection.
+   */
+  public constructor(
+    opts: ConnectionOpts,
+    sharedE2eeOption: E2EEOptions | undefined,
+  ) {
+    const factory =
+      opts.livekitRoomFactory ??
+      ((options: RoomOptions): LivekitRoom => new LivekitRoom(options));
+    const livekitRoom = factory({
+      ...defaultLiveKitOptions,
+      e2ee: sharedE2eeOption,
+    });
+    super(livekitRoom, opts);
+  }
+}

src/state/remoteMembers/displayname.ts

@@ -0,0 +1,78 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type Room, type RoomMember, RoomStateEvent } from "matrix-js-sdk";
+import { combineLatest, fromEvent, type Observable, startWith } from "rxjs";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { logger } from "matrix-js-sdk/lib/logger";
+import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
+
+import { type ObservableScope } from "../ObservableScope";
+import { calculateDisplayName, shouldDisambiguate } from "../../utils/displayname";
+
+/**
+ * Displayname for each member of the call. This will disambiguate
+ * any displayname that clashes with another member. Only members
+ * joined to the call are considered here.
+ */
+// don't do this work more times than we need to. This is achieved by converting to a behavior:
+export const memberDisplaynames$ = (
+  matrixRoom: Room,
+  memberships$: Observable<CallMembership[]>,
+  scope: ObservableScope,
+  userId: string,
+  deviceId: string,
+) =>
+  scope.behavior(
+    combineLatest(
+      [
+        // Handle call membership changes
+        memberships$,
+        // Additionally handle display name changes (implicitly reacting to them)
+        fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)),
+        // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
+      ],
+      (memberships, _displaynames) => {
+        const displaynameMap = new Map<string, string>([
+          [
+            `${userId}:${deviceId}`,
+            matrixRoom.getMember(userId)?.rawDisplayName ?? userId,
+          ],
+        ]);
+        const room = matrixRoom;
+
+        // We only consider RTC members for disambiguation as they are the only visible members.
+        for (const rtcMember of memberships) {
+          const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`;
+          const { member } = getRoomMemberFromRtcMember(rtcMember, room);
+          if (!member) {
+            logger.error(
+              "Could not find member for media id:",
+              matrixIdentifier,
+            );
+            continue;
+          }
+          const disambiguate = shouldDisambiguate(member, memberships, room);
+          displaynameMap.set(
+            matrixIdentifier,
+            calculateDisplayName(member, disambiguate),
+          );
+        }
+        return displaynameMap;
+      },
+    ),
+  );
+
+export function getRoomMemberFromRtcMember(
+  rtcMember: CallMembership,
+  room: MatrixRoom,
+): { id: string; member: RoomMember | undefined } {
+  return {
+    id: rtcMember.userId + ":" + rtcMember.deviceId,
+    member: room.getMember(rtcMember.userId) ?? undefined,
+  };
+}

src/state/remoteMembers/matrixLivekitMerger.ts

@@ -0,0 +1,199 @@
+/*
+Copyright 2025 Element c.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  LocalParticipant,
+  Participant,
+  RemoteParticipant,
+  type Participant as LivekitParticipant,
+  type Room as LivekitRoom,
+} from "livekit-client";
+import {
+  type MatrixRTCSession,
+  MatrixRTCSessionEvent,
+  type CallMembership,
+  type Transport,
+  LivekitTransport,
+  isLivekitTransport,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  combineLatest,
+  fromEvent,
+  map,
+  startWith,
+  switchMap,
+  type Observable,
+} from "rxjs";
+
+import { type ObservableScope } from "../ObservableScope";
+import { type Connection } from "./Connection";
+import { Behavior } from "../Behavior";
+import { RoomMember } from "matrix-js-sdk";
+import { getRoomMemberFromRtcMember } from "./displayname";
+
+
+// TODOs:
+// - make ConnectionManager its own actual class
+// - write test for scopes (do we really need to bind scope)
+class ConnectionManager {
+  constructor(transports$: Observable<Transport[]>) {}
+  public readonly connections$: Observable<Connection[]>;
+}
+
+/**
+ * Represent a matrix call member and his associated livekit participation.
+ * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
+ * or if it has no livekit transport at all.
+ */
+
+export interface MatrixLivekitItem {
+  callMembership: CallMembership;
+  livekitParticipant?: LivekitParticipant;
+}
+
+// Alternative structure idea:
+// const livekitMatrixItems$ = (callMemberships$,connectionManager,scope): Observable<MatrixLivekitItem[]> => {
+
+  // Map of Connection -> to (callMembership, LivekitParticipant?))
+type participants = {participant: LocalParticipant | RemoteParticipant}[]
+
+interface LivekitRoomWithParticipants {
+  livekitRoom: LivekitRoom;
+  url: string; // Included for use as a React key
+  participants: {
+    // What id is that??
+    // Looks like it userId:Deviceid?
+    id: string;
+    participant: LocalParticipant | RemoteParticipant | undefined;
+    // Why do we fetch a full room member here?
+    // looks like it is only for avatars?
+    // TODO: Remove that. have some Avatar Provider that can fetch avatar for user ids.
+    member: RoomMember;
+  }[];
+}
+
+/**
+ * Combines MatrixRtc and Livekit worlds.
+ *
+ * It has a small public interface:
+ *  - in (via constructor):
+ *    - an observable of CallMembership[] to track the call members (The matrix side)
+ *    - a `ConnectionManager` for the lk rooms (The livekit side)
+ *  - out (via public Observable):
+ *    - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data.
+ */
+export class MatrixLivekitMerger {
+  public remoteMatrixLivekitItems$: Observable<MatrixLivekitItem[]>;
+
+  /**
+   * The MatrixRTC session participants.
+   */
+  // Note that MatrixRTCSession already filters the call memberships by users
+  // that are joined to the room; we don't need to perform extra filtering here.
+  private readonly memberships$ = this.scope.behavior(
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(
+      startWith(null),
+      map(() => this.matrixRTCSession.memberships),
+    ),
+  );
+
+  public constructor(
+    private matrixRTCSession: MatrixRTCSession,
+    private connectionManager: ConnectionManager,
+    private scope: ObservableScope,
+  ) {
+    const publishingParticipants$ = combineLatest([
+      this.memberships$,
+      connectionManager.connections$,
+    ]).pipe(map(), this.scope.bind());
+    this.remoteMatrixLivekitItems$ = combineLatest([
+      callMemberships$,
+      connectionManager.connections$,
+    ]).pipe(this.scope.bind());
+    // Implementation goes here
+  }
+
+  /**
+   * Lists the transports used by ourselves, plus all other MatrixRTC session
+   * members. For completeness this also lists the preferred transport and
+   * whether we are in multi-SFU mode or sticky events mode (because
+   * advertisedTransport$ wants to read them at the same time, and bundling data
+   * together when it might change together is what you have to do in RxJS to
+   * avoid reading inconsistent state or observing too many changes.)
+   */
+  private readonly membershipsWithTransport$: Behavior<{
+    membership: CallMembership;
+    transport?: LivekitTransport;
+  } | null> = this.scope.behavior(
+    this.memberships$.pipe(
+      map((memberships) => {
+        const oldestMembership = this.matrixRTCSession.getOldestMembership();
+
+        memberships.map((membership) => {
+          let transport = membership.getTransport(oldestMembership ?? membership)
+          return { membership, transport: isLivekitTransport(transport) ? transport : undefined };
+        })
+      }),
+    ),
+  );
+
+  /**
+   * Lists the transports used by each MatrixRTC session member other than
+   * ourselves.
+   */
+  // private readonly remoteTransports$ = this.scope.behavior(
+  //   this.membershipsWithTransport$.pipe(
+  //     map((transports) => transports?.remote ?? []),
+  //   ),
+  // );
+
+  /**
+   * Lists, for each LiveKit room, the LiveKit participants whose media should
+   * be presented.
+   */
+  private readonly participantsByRoom$ = this.scope.behavior<LivekitRoomWithParticipants[]>(
+    // TODO: Move this logic into Connection/PublishConnection if possible
+
+           this.connectionManager.connections$.pipe(
+            switchMap((connections) => {
+                  connections.map((c)=>c.publishingParticipants$.pipe(
+                    map((publishingParticipants) => {
+                      const participants: {
+                        id: string;
+                        participant: LivekitParticipant | undefined;
+                        member: RoomMember;
+                      }[] = publishingParticipants.map(({ participant, membership }) => ({
+                        // TODO update to UUID
+                        id: `${membership.userId}:${membership.deviceId}`,
+                        participant,
+                        // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
+                        member:
+                          getRoomMemberFromRtcMember(
+                            membership,
+                            this.matrixRoom,
+                          )?.member ?? memberError(),
+                      }));
+
+                      return {
+                        livekitRoom: c.livekitRoom,
+                        url: c.transport.livekit_service_url,
+                        participants,
+                      };
+                    }),
+                  ),
+                ),
+              ),
+            ),
+          );
+        }),
+      )
+      .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)),
+  );
+}