Merge branch 'livekit' into toger5/delayed-event-delegation

.eslintrc.cjs

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
 `;
 
 module.exports = {
-  plugins: ["matrix-org", "rxjs"],
+  plugins: ["matrix-org", "rxjs", "jsdoc"],
   extends: [
     "plugin:matrix-org/react",
     "plugin:matrix-org/a11y",
@@ -26,6 +26,13 @@ module.exports = {
     node: true,
   },
   rules: {
+    "jsdoc/no-types": "error",
+    "jsdoc/empty-tags": "error",
+    "jsdoc/check-property-names": "error",
+    "jsdoc/check-values": "error",
+    "jsdoc/check-param-names": "warn",
+    // "jsdoc/require-param": "warn",
+    "jsdoc/require-param-description": "warn",
     "matrix-org/require-copyright-header": ["error", COPYRIGHT_HEADER],
     "jsx-a11y/media-has-caption": "off",
     "react/display-name": "error",
@@ -75,6 +82,23 @@ module.exports = {
         "no-console": ["error"],
       },
     },
+    {
+      files: [
+        "**/*.test.ts",
+        "**/*.test.tsx",
+        "**/test.ts",
+        "**/test.tsx",
+        "**/test-**",
+      ],
+      rules: {
+        "jsdoc/no-types": "off",
+        "jsdoc/empty-tags": "off",
+        "jsdoc/check-property-names": "off",
+        "jsdoc/check-values": "off",
+        "jsdoc/check-param-names": "off",
+        "jsdoc/require-param-description": "off",
+      },
+    },
   ],
   settings: {
     react: {

.github/workflows/publish.yaml

@@ -61,6 +61,7 @@ jobs:
       docker_tags: |
         type=sha,format=short,event=branch
         type=raw,value=${{ github.event.release.tag_name }}
+        type=raw,value=latest
       # Like before, using ${{ env.VERSION }} above doesn't work
   add_docker_release_note:
     needs: publish_docker

knip.ts

@@ -9,7 +9,7 @@ import { type KnipConfig } from "knip";
 
 export default {
   vite: {
-    config: ["vite.config.ts", "vite-embedded.config.ts"],
+    config: ["vite.config.ts", "vite-embedded.config.ts", "vite-sdk.config.ts"],
   },
   entry: ["src/main.tsx", "i18next-parser.config.ts"],
   ignoreBinaries: [

package.json

@@ -13,6 +13,8 @@
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
     "build:embedded:production": "yarn build:embedded",
     "build:embedded:development": "yarn build:embedded --mode development",
+    "build:sdk": "yarn build:full --config vite-sdk.config.js",
+    "build:sdk:development": "yarn build:sdk --mode development",
     "serve": "vite preview",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",
@@ -93,6 +95,7 @@
     "eslint-config-prettier": "^10.0.0",
     "eslint-plugin-deprecate": "^0.8.2",
     "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-jsdoc": "^61.5.0",
     "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-matrix-org": "2.1.0",
     "eslint-plugin-react": "^7.29.4",
@@ -111,6 +114,7 @@
     "loglevel": "^1.9.1",
     "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e",
     "matrix-widget-api": "^1.14.0",
+    "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",
     "pako": "^2.0.4",
@@ -133,6 +137,7 @@
     "vite": "^7.0.0",
     "vite-plugin-generate-file": "^0.3.0",
     "vite-plugin-html": "^3.2.2",
+    "vite-plugin-node-stdlib-browser": "^0.2.1",
     "vite-plugin-svgr": "^4.0.0",
     "vitest": "^3.0.0",
     "vitest-axe": "^1.0.0-pre.3"

playwright/errors.spec.ts

@@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details.
 
 import { expect, test } from "@playwright/test";
 
+import { createJTWToken } from "./fixtures/jwt-token";
+
 test("Should show error screen if fails to get JWT token", async ({ page }) => {
   await page.goto("/");
 
@@ -93,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({
         contentType: "application/json",
         body: JSON.stringify({
           url: "wss://badurltotricktest/livekit/sfu",
-          jwt: "FAKE",
+          jwt: createJTWToken("@fake:user", "!fake:room"),
         }),
       }),
   );

playwright/fixtures/jwt-token.ts

@@ -0,0 +1,22 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+export function createJTWToken(sub: string, room: string): string {
+  return [
+    {}, // header
+    {
+      // payload
+      sub,
+      video: {
+        room,
+      },
+    },
+    {}, // signature
+  ]
+    .map((d) => global.btoa(JSON.stringify(d)))
+    .join(".");
+}

playwright/fixtures/widget-user.ts

@@ -67,7 +67,6 @@ const CONFIG_JSON = {
 
 /**
  * Set the Element Call URL in the dev tool settings using `window.mxSettingsStore` via `page.evaluate`.
- * @param page
  */
 const setDevToolElementCallDevUrl = process.env.USE_DOCKER
   ? async (page: Page): Promise<void> => {

playwright/sfu-reconnect-bug.spec.ts

@@ -68,11 +68,6 @@ test("When creator left, avoid reconnect to the same SFU", async ({
     reducedMotion: "reduce",
   });
   const guestCPage = await guestC.newPage();
-  let sfuGetCallCount = 0;
-  await guestCPage.route("**/livekit/jwt/sfu/get", async (route) => {
-    sfuGetCallCount++;
-    await route.continue();
-  });
   // Track WebSocket connections
   let wsConnectionCount = 0;
   await guestCPage.routeWebSocket("**", (ws) => {
@@ -100,5 +95,4 @@ test("When creator left, avoid reconnect to the same SFU", async ({
   // https://github.com/element-hq/element-call/issues/3344
   // The app used to request a new jwt token then to reconnect to the SFU
   expect(wsConnectionCount).toBe(1);
-  expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */);
 });

sdk/README.md

@@ -0,0 +1,35 @@
+# SDK mode
+
+EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
+
+It allows to use matrixRTC in combination with livekit without relying on element call.
+
+This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods.
+
+This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
+
+## Widgets
+
+The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ...
+To create a widget see the example index.html file in this folder. And add it to EW via:
+`/addwidget <widgetUrl>` (see **url parameters** for more details on `<widgetUrl>`)
+
+### url parameters
+
+```
+widgetId = $matrix_widget_id
+perParticipantE2EE = true
+userId = $matrix_user_id
+deviceId = $org.matrix.msc3819.matrix_device_id
+baseUrl = $org.matrix.msc4039.matrix_base_url
+```
+
+`parentUrl = // will be inserted automatically`
+
+Full template use as `<widgetUrl>`:
+
+```
+http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
+```
+
+the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`)

sdk/helper.ts

@@ -0,0 +1,55 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * This file contains helper functions and types for the MatrixRTC SDK.
+ */
+
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
+import { scan } from "rxjs";
+
+import { widget as _widget } from "../src/widget";
+import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
+
+export const logger = rootLogger.getChild("[MatrixRTCSdk]");
+
+if (!_widget) throw Error("No widget. This webapp can only start as a widget");
+export const widget = _widget;
+
+export const tryMakeSticky = (): void => {
+  logger.info("try making sticky MatrixRTCSdk");
+  void widget.api
+    .setAlwaysOnScreen(true)
+    .then(() => {
+      logger.info("sticky MatrixRTCSdk");
+    })
+    .catch((error) => {
+      logger.error("failed to make sticky MatrixRTCSdk", error);
+    });
+};
+export const TEXT_LK_TOPIC = "matrixRTC";
+/**
+ * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
+ *
+ * I think there should be a builtin for this but i did not find it...
+ */
+export const currentAndPrev = scan<
+  LivekitRoomItem[],
+  {
+    prev: LivekitRoomItem[];
+    current: LivekitRoomItem[];
+  }
+>(
+  ({ current: lastCurrentVal }, items) => ({
+    prev: lastCurrentVal,
+    current: items,
+  }),
+  {
+    prev: [],
+    current: [],
+  },
+);

sdk/index.html

@@ -0,0 +1,87 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Godot MatrixRTC Widget</title>
+    <meta charset="utf-8" />
+    <script type="module">
+      // TODO use the url where the matrixrtc-sdk.js file from dist is hosted
+      import { createMatrixRTCSdk } from "http://localhost:8123/matrixrtc-sdk.js";
+
+      try {
+        window.matrixRTCSdk = await createMatrixRTCSdk(
+          "com.github.toger5.godot-game",
+        );
+        console.info("createMatrixRTCSdk was created!");
+      } catch (e) {
+        console.error("createMatrixRTCSdk", e);
+      }
+      const sdk = window.matrixRTCSdk;
+
+      // This is the main bridging interface to godot
+      window.matrixRTCSdkGodot = {
+        dataObs: sdk.data$,
+        memberObs: sdk.members$,
+        // join: sdk.join, // lets stick with autojoin for now
+        sendData: sdk.sendData,
+        leave: sdk.leave,
+        connectedObs: sdk.connected$,
+      };
+
+      console.info("matrixRTCSdk join ", sdk);
+      const connectionState = sdk.join();
+      console.info("matrixRTCSdk joined");
+
+      const div = document.getElementById("data");
+      div.innerHTML = "<h3>Data:</h3>";
+
+      sdk.data$.subscribe((data) => {
+        const child = document.createElement("p");
+        child.innerHTML = JSON.stringify(data);
+        div.appendChild(child);
+        // TODO forward to godot
+      });
+
+      sdk.members$.subscribe((memberObjects) => {
+        // reset div
+        const div = document.getElementById("members");
+        div.innerHTML = "<h3>Members:</h3>";
+
+        // create member list
+        const members = memberObjects.map((member) => member.membership.sender);
+        console.info("members changed", members);
+        for (const m of members) {
+          console.info("member", m);
+          const child = document.createElement("p");
+          child.innerHTML = m;
+          div.appendChild(child);
+        }
+      });
+
+      sdk.connected$.subscribe((connected) => {
+        console.info("connected changed", connected);
+        const div = document.getElementById("connect_status");
+        div.innerHTML = connected ? "Connected" : "Disconnected";
+      });
+
+      let engine = new Engine($GODOT_CONFIG);
+      engine.startGame();
+    </script>
+    <!--// TODO use it as godot HTML template-->
+    <script src="$GODOT_URL"></script>
+  </head>
+  <body>
+    <canvas id="canvas"></canvas>
+    <div
+      id="overlay"
+      style="position: absolute; top: 0; right: 0; background-color: #ffffff10"
+    >
+      <div id="connect_status"></div>
+      <button onclick="window.matrixRTCSdk.leave();">Leave</button>
+      <button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});">
+        Send Text
+      </button>
+      <div id="members"></div>
+      <div id="data"></div>
+    </div>
+  </body>
+</html>

sdk/main.ts

@@ -0,0 +1,308 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * This file is the entrypoint for the sdk build of element call: `yarn build:sdk`
+ * use in widgets.
+ * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
+ * can be used to join a rtc session and exchange realtime data.
+ * It takes care of all the tricky bits:
+ *  - sending delayed events
+ *  - finding the right sfu
+ *  - handling the media stream
+ *  - sending join/leave state or sticky events
+ *  - setting up encryption and scharing keys
+ */
+
+import {
+  combineLatest,
+  map,
+  type Observable,
+  of,
+  shareReplay,
+  Subject,
+  switchMap,
+  tap,
+} from "rxjs";
+import {
+  type CallMembership,
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  type Room as LivekitRoom,
+  type TextStreamReader,
+  type LocalParticipant,
+  type RemoteParticipant,
+} from "livekit-client";
+
+// TODO how can this get fixed? to just be part of `livekit-client`
+// Can this be done in the tsconfig.json
+import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types";
+import { type Behavior, constant } from "../src/state/Behavior";
+import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel";
+import { ObservableScope } from "../src/state/ObservableScope";
+import { getUrlParams } from "../src/UrlParams";
+import { MuteStates } from "../src/state/MuteStates";
+import { MediaDevices } from "../src/state/MediaDevices";
+import { E2eeType } from "../src/e2ee/e2eeType";
+import {
+  currentAndPrev,
+  logger,
+  TEXT_LK_TOPIC,
+  tryMakeSticky,
+  widget,
+} from "./helper";
+import { ElementWidgetActions } from "../src/widget";
+import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
+
+interface MatrixRTCSdk {
+  /**
+   * observe connected$ to track the state.
+   * @returns
+   */
+  join: () => void;
+  /** @throws on leave errors */
+  leave: () => void;
+  data$: Observable<{ sender: string; data: string }>;
+  /**
+   * flattened list of members
+   */
+  members$: Behavior<
+    {
+      connection: Connection | null;
+      membership: CallMembership;
+      participant: LocalParticipant | RemoteParticipant | null;
+    }[]
+  >;
+  /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
+  connected$: Behavior<boolean>;
+  sendData?: (data: unknown) => Promise<void>;
+}
+
+export async function createMatrixRTCSdk(
+  application: string = "m.call",
+  id: string = "",
+): Promise<MatrixRTCSdk> {
+  logger.info("Hello");
+  const client = await widget.client;
+  logger.info("client created");
+  const scope = new ObservableScope();
+  const { roomId } = getUrlParams();
+  if (roomId === null) throw Error("could not get roomId from url params");
+
+  const room = client.getRoom(roomId);
+  if (room === null) throw Error("could not get room from client");
+
+  const mediaDevices = new MediaDevices(scope);
+  const muteStates = new MuteStates(scope, mediaDevices, constant(true));
+  const slot = { application, id };
+  const rtcSession = new MatrixRTCSession(
+    client,
+    room,
+    MatrixRTCSession.sessionMembershipsForSlot(room, slot),
+    slot,
+  );
+  const callViewModel = createCallViewModel$(
+    scope,
+    rtcSession,
+    room,
+    mediaDevices,
+    muteStates,
+    { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } },
+    of({}),
+    of({}),
+    constant({ supported: false, processor: undefined }),
+  );
+  logger.info("CallViewModelCreated");
+  // create data listener
+  const data$ = new Subject<{ sender: string; data: string }>();
+
+  const lkTextStreamHandlerFunction = async (
+    reader: TextStreamReader,
+    participantInfo: { identity: string },
+    livekitRoom: LivekitRoom,
+  ): Promise<void> => {
+    const info = reader.info;
+    logger.info(
+      `Received text stream from ${participantInfo.identity}\n` +
+        `  Topic: ${info.topic}\n` +
+        `  Timestamp: ${info.timestamp}\n` +
+        `  ID: ${info.id}\n` +
+        `  Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
+    );
+
+    const participants = callViewModel.livekitRoomItems$.value.find(
+      (i) => i.livekitRoom === livekitRoom,
+    )?.participants;
+    if (participants && participants.includes(participantInfo.identity)) {
+      const text = await reader.readAll();
+      logger.info(`Received text: ${text}`);
+      data$.next({ sender: participantInfo.identity, data: text });
+    } else {
+      logger.warn(
+        "Received text from unknown participant",
+        participantInfo.identity,
+      );
+    }
+  };
+
+  const livekitRoomItemsSub = callViewModel.livekitRoomItems$
+    .pipe(
+      tap((beforecurrentAndPrev) => {
+        logger.info(
+          `LiveKit room items updated: ${beforecurrentAndPrev.length}`,
+          beforecurrentAndPrev,
+        );
+      }),
+      currentAndPrev,
+      tap((aftercurrentAndPrev) => {
+        logger.info(
+          `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
+          aftercurrentAndPrev,
+        );
+      }),
+    )
+    .subscribe({
+      next: ({ prev, current }) => {
+        const prevRooms = prev.map((i) => i.livekitRoom);
+        const currentRooms = current.map((i) => i.livekitRoom);
+        const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
+        const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
+        addedRooms.forEach((r) => {
+          logger.info(`Registering text stream handler for room `);
+          r.registerTextStreamHandler(
+            TEXT_LK_TOPIC,
+            (reader, participantInfo) =>
+              void lkTextStreamHandlerFunction(reader, participantInfo, r),
+          );
+        });
+        removedRooms.forEach((r) => {
+          logger.info(`Unregistering text stream handler for room `);
+          r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
+        });
+      },
+      complete: () => {
+        logger.info("Livekit room items subscription completed");
+        for (const item of callViewModel.livekitRoomItems$.value) {
+          logger.info("unregistering room item from room", item.url);
+          item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
+        }
+      },
+    });
+
+  // create sendData function
+  const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
+    scope.behavior(
+      callViewModel.localMatrixLivekitMember$.pipe(
+        switchMap((m) => {
+          if (!m)
+            return of((data: string): never => {
+              throw Error("local membership not yet ready.");
+            });
+          return m.participant.value$.pipe(
+            map((p) => {
+              if (p === null) {
+                return (data: string): never => {
+                  throw Error("local participant not yet ready to send data.");
+                };
+              } else {
+                return async (data: string): Promise<TextStreamInfo> =>
+                  p.sendText(data, { topic: TEXT_LK_TOPIC });
+              }
+            }),
+          );
+        }),
+      ),
+    );
+
+  const sendData = async (data: unknown): Promise<void> => {
+    const dataString = JSON.stringify(data);
+    logger.info("try sending: ", dataString);
+    try {
+      await Promise.resolve();
+      const info = await sendFn.value(dataString);
+      logger.info(`Sent text with stream ID: ${info.id}`);
+    } catch (e) {
+      logger.error("failed sending: ", dataString, e);
+    }
+  };
+
+  // after hangup gets called
+  const leaveSubs = callViewModel.leave$.subscribe(() => {
+    const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
+      const leaveResolver = Promise.withResolvers<void>();
+      logger.info("waiting for RTC leave");
+      rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
+        logger.info("received RTC join update: ", isJoined);
+        if (!isJoined) leaveResolver.resolve();
+      });
+      await leaveResolver.promise;
+      logger.info("send Unstick");
+      await widget.api
+        .setAlwaysOnScreen(false)
+        .catch((e) =>
+          logger.error(
+            "Failed to set call widget `alwaysOnScreen` to false",
+            e,
+          ),
+        );
+      logger.info("send Close");
+      await widget.api.transport
+        .send(ElementWidgetActions.Close, {})
+        .catch((e) => logger.error("Failed to send close action", e));
+    };
+
+    // schedule close first and then leave (scope.end)
+    void scheduleWidgetCloseOnLeave();
+
+    // actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
+    scope.end();
+  });
+
+  logger.info("createMatrixRTCSdk done");
+
+  return {
+    join: (): void => {
+      // first lets try making the widget sticky
+      tryMakeSticky();
+      callViewModel.join();
+    },
+    leave: (): void => {
+      callViewModel.hangup();
+      leaveSubs.unsubscribe();
+      livekitRoomItemsSub.unsubscribe();
+    },
+    data$,
+    connected$: callViewModel.connected$,
+    members$: scope.behavior(
+      callViewModel.matrixLivekitMembers$.pipe(
+        switchMap((members) => {
+          const listOfMemberObservables = members.map((member) =>
+            combineLatest([
+              member.connection$,
+              member.membership$,
+              member.participant.value$,
+            ]).pipe(
+              map(([connection, membership, participant]) => ({
+                connection,
+                membership,
+                participant,
+              })),
+              // using shareReplay instead of a Behavior here because the behavior would need
+              // a tricky scope.end() setup.
+              shareReplay({ bufferSize: 1, refCount: true }),
+            ),
+          );
+          return combineLatest(listOfMemberObservables);
+        }),
+      ),
+      [],
+    ),
+    sendData,
+  };
+}

src/e2ee/sharedKeyManagement.ts

@@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
   `room-shared-key-${roomId}`;
 
 /**
- * An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`.
+ * An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`.
- * @param roomId
+ * @param roomId The room ID we want the shared key for.
  * @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this.
  * @returns [roomSharedKey, setRoomSharedKey] like a react useState hook.
  */

src/livekit/MatrixAudioRenderer.tsx

@@ -165,7 +165,11 @@ interface StereoPanAudioTrackProps {
  * It main purpose is to remount the AudioTrack component when switching from
  * audioContext to normal audio playback.
  * As of now the AudioTrack component does not support adding audio nodes while being mounted.
- * @param param0
+ * @param props The component props
+ * @param props.trackRef The track reference
+ * @param props.muted If the track should be muted
+ * @param props.audioContext The audio context to use
+ * @param props.audioNodes The audio nodes to use
  * @returns
  */
 function AudioTrackWithAudioNodes({

src/livekit/openIDSFU.test.ts

@@ -0,0 +1,112 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  beforeEach,
+  afterEach,
+  describe,
+  expect,
+  it,
+  type MockedObject,
+  vitest,
+} from "vitest";
+import fetchMock from "fetch-mock";
+
+import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
+import { testJWTToken } from "../utils/test-fixtures";
+
+const sfuUrl = "https://sfu.example.org";
+
+describe("getSFUConfigWithOpenID", () => {
+  let matrixClient: MockedObject<OpenIDClientParts>;
+  beforeEach(() => {
+    matrixClient = {
+      getOpenIdToken: vitest.fn(),
+      getDeviceId: vitest.fn(),
+    };
+  });
+  afterEach(() => {
+    vitest.clearAllMocks();
+    fetchMock.reset();
+  });
+  it("should handle fetching a token", async () => {
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 200,
+        body: { url: sfuUrl, jwt: testJWTToken },
+      };
+    });
+    const config = await getSFUConfigWithOpenID(
+      matrixClient,
+      "https://sfu.example.org",
+      "!example_room_id",
+    );
+    expect(config).toEqual({
+      jwt: testJWTToken,
+      url: sfuUrl,
+      livekitIdentity: "@me:example.org:ABCDEF",
+      livekitAlias: "!example_room_id",
+    });
+    void (await fetchMock.flush());
+  });
+  it("should fail if the SFU errors", async () => {
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 500,
+        body: { error: "Test failure" },
+      };
+    });
+    try {
+      await getSFUConfigWithOpenID(
+        matrixClient,
+        "https://sfu.example.org",
+        "!example_room_id",
+      );
+    } catch (ex) {
+      expect(((ex as Error).cause as Error).message).toEqual(
+        "SFU Config fetch failed with status code 500",
+      );
+      void (await fetchMock.flush());
+      return;
+    }
+    expect.fail("Expected test to throw;");
+  });
+
+  it("should retry fetching the openid token", async () => {
+    let count = 0;
+    matrixClient.getOpenIdToken.mockImplementation(async () => {
+      count++;
+      if (count < 2) {
+        throw Error("Test failure");
+      }
+      return Promise.resolve({
+        token_type: "Bearer",
+        access_token: "foobar",
+        matrix_server_name: "example.org",
+        expires_in: 30,
+      });
+    });
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 200,
+        body: { url: sfuUrl, jwt: testJWTToken },
+      };
+    });
+    const config = await getSFUConfigWithOpenID(
+      matrixClient,
+      "https://sfu.example.org",
+      "!example_room_id",
+    );
+    expect(config).toEqual({
+      jwt: testJWTToken,
+      url: sfuUrl,
+      livekitIdentity: "@me:example.org:ABCDEF",
+      livekitAlias: "!example_room_id",
+    });
+    void (await fetchMock.flush());
+  });
+});

src/livekit/openIDSFU.ts

@@ -13,9 +13,47 @@ import { FailToGetOpenIdToken } from "../utils/errors";
 import { doNetworkOperationWithRetry } from "../utils/matrix";
 import { Config } from "../config/Config";
 
+/**
+ * Configuration and access tokens provided by the SFU on successful authentication.
+ */
 export interface SFUConfig {
   url: string;
   jwt: string;
+  livekitAlias: string;
+  livekitIdentity: string;
+}
+
+/**
+ * Decoded details from the JWT.
+ */
+interface SFUJWTPayload {
+  /**
+   * Expiration time for the JWT.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  exp: number;
+  /**
+   * Name of the instance which authored the JWT
+   */
+  iss: string;
+  /**
+   * Time at which the JWT can start to be used.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  nbf: number;
+  /**
+   * Subject. The Livekit alias in this context.
+   */
+  sub: string;
+  /**
+   * The set of permissions for the user.
+   */
+  video: {
+    canPublish: boolean;
+    canSubscribe: boolean;
+    room: string;
+    roomJoin: boolean;
+  };
 }
 
 // The bits we need from MatrixClient
@@ -27,15 +65,15 @@ export type OpenIDClientParts = Pick<
  * Gets a bearer token from the homeserver and then use it to authenticate
  * to the matrix RTC backend in order to get acces to the SFU.
  * It has built-in retry for calls to the homeserver with a backoff policy.
- * @param client
+ * @param client The Matrix client
  * @param membership
- * @param serviceUrl
+ * @param serviceUrl The URL of the livekit SFU service
  * @param forceOldEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination
  * instead of a hash.
  * This function by default uses whatever is possible with the current jwt service installed next to the SFU.
  * For remote connections this does not matter, since we will not publish there we can rely on the newest option.
  * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events.
- * @param livekitRoomAlias
+ * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases.
  * @param delayEndpointBaseUrl
  * @param delayId
  * @param logger
@@ -47,7 +85,7 @@ export async function getSFUConfigWithOpenID(
   membership: CallMembershipIdentityParts,
   serviceUrl: string,
   forceOldJwtEndpoint: boolean,
-  livekitRoomAlias: string,
+  roomId: string,
   delayEndpointBaseUrl?: string,
   delayId?: string,
   logger?: Logger,
@@ -68,39 +106,49 @@ export async function getSFUConfigWithOpenID(
   const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [
     membership,
     serviceUrl,
-    livekitRoomAlias,
+    roomId,
     openIdToken,
   ];
+
+  let sfuConfig: { url: string; jwt: string };
   try {
     // we do not want to try the old endpoint, since we are not sending the new matrix2.0 sticky events (no hashed identity in the event)
     if (forceOldJwtEndpoint) throw new Error("Force old jwt endpoint");
     if (!delayId)
       throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint.");
 
-    const sfuConfig = await getLiveKitJWTWithDelayDelegation(
+    sfuConfig = await getLiveKitJWTWithDelayDelegation(
       ...args,
       delayEndpointBaseUrl,
       delayId,
     );
     logger?.info(`Got JWT from call's active focus URL.`);
-    return sfuConfig;
   } catch (e) {
     logger?.warn(
       `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`,
       e,
     );
-    const sfuConfig = await getLiveKitJWT(...args);
+    sfuConfig = await getLiveKitJWT(...args);
     logger?.info(`Got JWT from call's active focus URL.`);
-    return sfuConfig;
+  } // Pull the details from the JWT
-  }
+  const [, payloadStr] = sfuConfig.jwt.split(".");
+
+  const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
+  return {
+    jwt: sfuConfig.jwt,
+    url: sfuConfig.url,
+    livekitAlias: payload.video.room,
+    // NOTE: Currently unused.
+    livekitIdentity: payload.sub,
+  };
 }
 
 async function getLiveKitJWT(
   membership: CallMembershipIdentityParts,
   livekitServiceURL: string,
-  livekitRoomAlias: string,
+  matrixRoomId: string,
   openIDToken: IOpenIDToken,
-): Promise<SFUConfig> {
+): Promise<{ url: string; jwt: string }> {
   try {
     const res = await fetch(livekitServiceURL + "/sfu/get", {
       method: "POST",
@@ -108,7 +156,8 @@ async function getLiveKitJWT(
         "Content-Type": "application/json",
       },
       body: JSON.stringify({
-        room: livekitRoomAlias,
+        // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used.
+        room: matrixRoomId,
         openid_token: openIDToken,
         device_id: membership.deviceId,
       }),
@@ -118,22 +167,22 @@ async function getLiveKitJWT(
     }
     return await res.json();
   } catch (e) {
-    throw new Error("SFU Config fetch failed with exception " + e);
+    throw new Error("SFU Config fetch failed with exception", { cause: e });
   }
 }
 
 export async function getLiveKitJWTWithDelayDelegation(
   membership: CallMembershipIdentityParts,
   livekitServiceURL: string,
-  livekitRoomAlias: string,
+  matrixRoomId: string,
   openIDToken: IOpenIDToken,
   delayEndpointBaseUrl?: string,
   delayId?: string,
-): Promise<SFUConfig> {
+): Promise<{ url: string; jwt: string }> {
   const { userId, deviceId, memberId } = membership;
 
   const body = {
-    room_id: livekitRoomAlias,
+    room_id: matrixRoomId,
     slot_id: "m.call#ROOM",
     openid_token: openIDToken,
     member: {

src/reactions/ReactionsReader.ts

@@ -135,10 +135,10 @@ export class ReactionsReader {
   }
 
   /**
-   * Fetchest any hand wave reactions by the given sender on the given
+   * Fetches any hand wave reactions by the given sender on the given
    * membership event.
-   * @param membershipEventId
+   * @param membershipEventId - The user membership event id.
-   * @param expectedSender
+   * @param expectedSender - The expected sender of the reaction.
    * @returns A MatrixEvent if one was found.
    */
   private getLastReactionEvent(

src/room/InCallView.tsx

@@ -260,7 +260,7 @@ export const InCallView: FC<InCallViewProps> = ({
     () => void toggleRaisedHand(),
   );
 
-  const audioParticipants = useBehavior(vm.audioParticipants$);
+  const audioParticipants = useBehavior(vm.livekitRoomItems$);
   const participantCount = useBehavior(vm.participantCount$);
   const reconnecting = useBehavior(vm.reconnecting$);
   const windowMode = useBehavior(vm.windowMode$);

src/room/useLoadGroupCall.ts

@@ -106,22 +106,18 @@ async function joinRoomAfterInvite(
 
 export class CallTerminatedMessage extends Error {
   /**
+   * Creates a new CallTerminatedMessage.
+   *
+   * @param icon The icon to display with the message
    * @param messageTitle The title of the call ended screen message (translated)
+   * @param messageBody The message explaining the kind of termination
+   * (kick, ban, knock reject, etc.) (translated)
+   * @param reason  The user-provided reason for the termination (kick/ban)
    */
   public constructor(
-    /**
-     * The icon to display with the message.
-     */
     public readonly icon: ComponentType<SVGAttributes<SVGElement>>,
     messageTitle: string,
-    /**
-     * The message explaining the kind of termination (kick, ban, knock reject,
-     * etc.) (translated)
-     */
     public readonly messageBody: string,
-    /**
-     * The user-provided reason for the termination (kick/ban)
-     */
     public readonly reason?: string,
   ) {
     super(messageTitle);

src/settings/rageshake.ts

@@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter {
 
   /**
    * Returns the log lines to flush to disk and empties the internal log buffer
-   * @return {string} \n delimited log lines
+   * @return \n delimited log lines
    */
   public popLogs(): string {
     const logsToFlush = this.logs;
@@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter {
 
   /**
    * Returns lines currently in the log buffer without removing them
-   * @return {string} \n delimited log lines
+   * @return \n delimited log lines
    */
   public peekLogs(): string {
     return this.logs;
@@ -139,7 +139,7 @@ class IndexedDBLogStore {
   }
 
   /**
-   * @return {Promise} Resolves when the store is ready.
+   * @return Resolves when the store is ready.
    */
   public async connect(): Promise<void> {
     const req = this.indexedDB.open("logs");
@@ -219,7 +219,7 @@ class IndexedDBLogStore {
    * This guarantees that we will always eventually do a flush when flush() is
    * called.
    *
-   * @return {Promise} Resolved when the logs have been flushed.
+   * @return Resolved when the logs have been flushed.
    */
   public flush = async (): Promise<void> => {
     // check if a flush() operation is ongoing
@@ -270,7 +270,7 @@ class IndexedDBLogStore {
    * returned are deleted at the same time, so this can be called at startup
    * to do house-keeping to keep the logs from growing too large.
    *
-   * @return {Promise<Object[]>} Resolves to an array of objects. The array is
+   * @return Resolves to an array of objects. The array is
    * sorted in time (oldest first) based on when the log file was created (the
    * log ID). The objects have said log ID in an "id" field and "lines" which
    * is a big string with all the new-line delimited logs.
@@ -421,12 +421,12 @@ class IndexedDBLogStore {
 
 /**
  * Helper method to collect results from a Cursor and promiseify it.
- * @param {ObjectStore|Index} store The store to perform openCursor on.
+ * @param store - The store to perform openCursor on.
- * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
+ * @param keyRange - Optional key range to apply on the cursor.
- * @param {Function} resultMapper A function which is repeatedly called with a
+ * @param resultMapper - A function which is repeatedly called with a
  * Cursor.
  * Return the data you want to keep.
- * @return {Promise<T[]>} Resolves to an array of whatever you returned from
+ * @return Resolves to an array of whatever you returned from
  * resultMapper.
  */
 async function selectQuery<T>(
@@ -464,9 +464,7 @@ declare global {
 /**
  * Configure rage shaking support for sending bug reports.
  * Modifies globals.
- * @param {boolean} setUpPersistence When true (default), the persistence will
+ * @return Resolves when set up.
- * be set up immediately for the logs.
- * @return {Promise} Resolves when set up.
  */
 export async function init(): Promise<void> {
   global.mx_rage_logger = new ConsoleLogger();
@@ -503,7 +501,7 @@ export async function init(): Promise<void> {
 /**
  * Try to start up the rageshake storage for logs. If not possible (client unsupported)
  * then this no-ops.
- * @return {Promise} Resolves when complete.
+ * @return Resolves when complete.
  */
 async function tryInitStorage(): Promise<void> {
   if (global.mx_rage_initStoragePromise) {
@@ -536,7 +534,7 @@ async function tryInitStorage(): Promise<void> {
 /**
  * Get a recent snapshot of the logs, ready for attaching to a bug report
  *
- * @return {LogEntry[]}  list of log data
+ * @return list of log data
  */
 export async function getLogsForReport(): Promise<LogEntry[]> {
   if (!global.mx_rage_logger) {

src/state/CallViewModel/CallNotificationLifecycle.ts

@@ -81,7 +81,7 @@ export interface Props {
   localUser: { deviceId: string; userId: string };
 }
 /**
- * @returns {callPickupState$, autoLeave$}
+ * @returns two observables:
  * `callPickupState$` The current call pickup state of the call.
  *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
  *     Then we can conclude if we were the first one to join or not.

src/state/CallViewModel/CallViewModel.ts

@@ -82,7 +82,7 @@ import {
 } from "../../reactions";
 import { shallowEquals } from "../../utils/array";
 import { type MediaDevices } from "../MediaDevices";
-import { type Behavior } from "../Behavior";
+import { constant, type Behavior } from "../Behavior";
 import { E2eeType } from "../../e2ee/e2eeType";
 import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
 import { type MuteStates } from "../MuteStates";
@@ -119,6 +119,7 @@ import {
   createMatrixLivekitMembers$,
   type TaggedParticipant,
   type LocalMatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./remoteMembers/MatrixLivekitMembers.ts";
 import {
   type AutoLeaveReason,
@@ -158,7 +159,7 @@ export interface CallViewModelOptions {
   /** Optional behavior overriding the computed window size, mainly for testing purposes. */
   windowSize$?: Behavior<{ width: number; height: number }>;
   /** The version & compatibility mode of MatrixRTC that we should use. */
-  matrixRTCMode$: Behavior<MatrixRTCMode>;
+  matrixRTCMode$?: Behavior<MatrixRTCMode>;
 }
 
 // Do not play any sounds if the participant count has exceeded this
@@ -184,7 +185,7 @@ interface LayoutScanState {
 }
 
 type MediaItem = UserMedia | ScreenShare;
-type AudioLivekitItem = {
+export type LivekitRoomItem = {
   livekitRoom: LivekitRoom;
   participants: string[];
   url: string;
@@ -207,8 +208,11 @@ export interface CallViewModel {
   callPickupState$: Behavior<
     "unknown" | "ringing" | "timeout" | "decline" | "success" | null
   >;
+  /** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
+   * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
+   */
   leave$: Observable<"user" | AutoLeaveReason>;
-  /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
+  /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
   hangup: () => void;
 
   // joining
@@ -260,7 +264,11 @@ export interface CallViewModel {
    */
   participantCount$: Behavior<number>;
   /** Participants sorted by livekit room so they can be used in the audio rendering */
-  audioParticipants$: Behavior<AudioLivekitItem[]>;
+  livekitRoomItems$: Behavior<LivekitRoomItem[]>;
+  userMedia$: Behavior<UserMedia[]>;
+  /** use the layout instead, this is just for the sdk export. */
+  matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
+  localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
   /** List of participants raising their hand */
   handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
   /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -343,17 +351,15 @@ export interface CallViewModel {
     switch: () => void;
   } | null>;
 
-  // connection state
   /**
-   * Whether various media/event sources should pretend to be disconnected from
+   * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
-   * all network input, even if their connection still technically works.
    */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
   reconnecting$: Behavior<boolean>;
+
+  /**
+   * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
+   */
+  connected$: Behavior<boolean>;
 }
 
 /**
@@ -386,6 +392,8 @@ export function createCallViewModel$(
     options.encryptionSystem,
     matrixRTCSession,
   );
+  const matrixRTCMode$ =
+    options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
 
   // Each hbar seperates a block of input variables required for the CallViewModel to function.
   // The outputs of this block is written under the hbar.
@@ -420,7 +428,7 @@ export function createCallViewModel$(
   };
 
   const useOldJwtEndpoint$ = scope.behavior(
-    options.matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
+    matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
   );
   const localTransport$ = createLocalTransport$({
     scope: scope,
@@ -439,7 +447,7 @@ export function createCallViewModel$(
     roomId: matrixRoom.roomId,
     useOldJwtEndpoint$,
     useOldestMember$: scope.behavior(
-      options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
+      matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
     ),
   });
 
@@ -482,7 +490,7 @@ export function createCallViewModel$(
   });
 
   const connectOptions$ = scope.behavior(
-    options.matrixRTCMode$.pipe(
+    matrixRTCMode$.pipe(
       map((mode) => ({
         encryptMedia: livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each change of matrixRTCMode...
@@ -515,7 +523,7 @@ export function createCallViewModel$(
         muteStates,
         trackProcessorState$,
         logger.getChild(
-          "[Publisher" + connection.transport.livekit_service_url + "]",
+          "[Publisher " + connection.transport.livekit_service_url + "]",
         ),
       );
     },
@@ -614,7 +622,7 @@ export function createCallViewModel$(
     ),
   );
 
-  const audioParticipants$ = scope.behavior(
+  const livekitRoomItems$ = scope.behavior(
     matrixLivekitMembers$.pipe(
       switchMap((members) => {
         const a$ = combineLatest(
@@ -639,7 +647,7 @@ export function createCallViewModel$(
         return a$;
       }),
       map((members) =>
-        members.reduce<AudioLivekitItem[]>((acc, curr) => {
+        members.reduce<LivekitRoomItem[]>((acc, curr) => {
           if (!curr) return acc;
 
           const existing = acc.find((item) => item.url === curr.url);
@@ -1509,10 +1517,7 @@ export function createCallViewModel$(
       ),
       null,
     ),
-
     participantCount$: participantCount$,
-    audioParticipants$: audioParticipants$,
-
     handsRaised$: handsRaised$,
     reactions$: reactions$,
     joinSoundEffect$: joinSoundEffect$,
@@ -1531,6 +1536,16 @@ export function createCallViewModel$(
     spotlight$: spotlight$,
     pip$: pip$,
     layout$: layout$,
+    userMedia$,
+    localMatrixLivekitMember$,
+    matrixLivekitMembers$: scope.behavior(
+      matrixLivekitMembers$.pipe(
+        map((members) => members.value),
+        tap((v) => {
+          logger.debug("matrixLivekitMembers$ updated (exported)", v);
+        }),
+      ),
+    ),
     tileStoreGeneration$: tileStoreGeneration$,
     showSpotlightIndicators$: showSpotlightIndicators$,
     showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1539,6 +1554,8 @@ export function createCallViewModel$(
     earpieceMode$: earpieceMode$,
     audioOutputSwitcher$: audioOutputSwitcher$,
     reconnecting$: localMembership.reconnecting$,
+    livekitRoomItems$,
+    connected$: localMembership.connected$,
   };
 }
 

src/state/CallViewModel/localMember/LocalMember.ts

@@ -139,7 +139,16 @@ interface Props {
  * We want
  *  - a publisher
  *  -
- * @param param0
+ * @param props The properties required to create the local membership.
+ * @param props.scope The observable scope to use.
+ * @param props.connectionManager The connection manager to get connections from.
+ * @param props.createPublisherFactory Factory to create a publisher once we have a connection.
+ * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
+ * @param props.homeserverConnected The homeserver connected state.
+ * @param props.localTransport$ The local transport to use for publishing.
+ * @param props.logger The logger to use.
+ * @param props.muteStates The mute states for video and audio.
+ * @param props.matrixRTCSession The matrix RTC session to join.
  * @returns
  *  - publisher: The handle to create tracks and publish them to the room.
  *  - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -178,14 +187,18 @@ export const createLocalMembership$ = ({
   // tracks$: Behavior<LocalTrack[]>;
   participant$: Behavior<LocalParticipant | null>;
   connection$: Behavior<Connection | null>;
-  /** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
+  /**
-   * Direct translation to the js-sdk membership manager connection `Status`.
+   * Tracks the homserver and livekit connected state and based on that computes reconnecting.
    */
   reconnecting$: Behavior<boolean>;
   /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
    * Direct translation to the js-sdk membership manager connection `Status`.
    */
   disconnected$: Behavior<boolean>;
+  /**
+   * Fully connected
+   */
+  connected$: Behavior<boolean>;
 } => {
   const logger = parentLogger.getChild("[LocalMembership]");
   logger.debug(`Creating local membership..`);
@@ -639,6 +652,7 @@ export const createLocalMembership$ = ({
     localMemberState$,
     participant$,
     reconnecting$,
+    connected$: matrixAndLivekitConnected$,
     disconnected$: scope.behavior(
       homeserverConnected.rtsSession$.pipe(
         map((state) => state === RTCSessionStatus.Disconnected),
@@ -672,9 +686,11 @@ interface EnterRTCSessionOptions {
  *      - Delay events management
  *      - Handles retries (fails only after several attempts)
  *
- * @param rtcSession
+ * @param rtcSession - The MatrixRTCSession to join.
- * @param transport
+ * @param transport - The LivekitTransport to use for this session.
- * @param options
+ * @param options - Options for entering the RTC session.
+ * @param options.encryptMedia - Whether to encrypt media.
+ * @param options.matrixRTCMode - The Matrix RTC mode to use.
  * @throws If the widget could not send ElementWidgetActions.JoinCall action.
  */
 // Exported for unit testing

src/state/CallViewModel/localMember/LocalTransport.test.ts

@@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  type MockedObject,
+  vi,
+} from "vitest";
 import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject } from "rxjs";
+import { BehaviorSubject, lastValueFrom } from "rxjs";
+import fetchMock from "fetch-mock";
 
 import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
 import { createLocalTransport$ } from "./LocalTransport";
@@ -18,8 +27,17 @@ import {
   FailToGetOpenIdToken,
 } from "../../../utils/errors";
 import * as openIDSFU from "../../../livekit/openIDSFU";
+import { customLivekitUrl } from "../../../settings/settings";
+import { testJWTToken } from "../../../utils/test-fixtures";
 
 describe("LocalTransport", () => {
+  const openIdResponse: openIDSFU.SFUConfig = {
+    url: "https://lk.example.org",
+    jwt: testJWTToken,
+    livekitAlias: "!example_room_id",
+    livekitIdentity: "@lk_user:ABCDEF",
+  };
+
   let scope: ObservableScope;
   beforeEach(() => (scope = new ObservableScope()));
   afterEach(() => scope.end());
@@ -65,13 +83,15 @@ describe("LocalTransport", () => {
     const errors: Error[] = [];
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(false),
       memberships$: constant(new Epoch<CallMembership[]>([])),
       client: {
         baseUrl: "https://lk.example.org",
         // Use empty domain to skip .well-known and use config directly
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
       },
@@ -145,11 +165,13 @@ describe("LocalTransport", () => {
 
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(true),
       memberships$,
       client: {
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
         baseUrl: "https://lk.example.org",
@@ -159,14 +181,159 @@ describe("LocalTransport", () => {
       delayId$: constant("delay_id_mock"),
     });
 
-    openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
+    openIdResolver.resolve?.(openIdResponse);
     expect(localTransport$.value).toBe(null);
     await flushPromises();
     // final
     expect(localTransport$.value).toStrictEqual({
-      livekit_alias: "!room:example.org",
+      livekit_alias: "!example_room_id",
       livekit_service_url: "https://lk.example.org",
       type: "livekit",
     });
   });
+
+  type LocalTransportProps = Parameters<typeof createLocalTransport$>[0];
+
+  describe("transport configuration mechanisms", () => {
+    let localTransportOpts: LocalTransportProps & {
+      client: MockedObject<LocalTransportProps["client"]>;
+    };
+    let openIdResolver: PromiseWithResolvers<openIDSFU.SFUConfig>;
+    beforeEach(() => {
+      mockConfig({});
+      customLivekitUrl.setValue(customLivekitUrl.defaultValue);
+      localTransportOpts = {
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: vi.fn().mockReturnValue(""),
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      };
+      openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
+      vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
+        openIdResolver.promise,
+      );
+    });
+
+    afterEach(() => {
+      fetchMock.reset();
+    });
+
+    it("supports getting transport via application config", async () => {
+      mockConfig({
+        livekit: { livekit_service_url: "https://lk.example.org" },
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via user settings", async () => {
+      customLivekitUrl.setValue("https://lk.example.org");
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via backend", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("fails fast if the openID request fails for backend config", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("supports getting transport via well-known", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+      expect(fetchMock.done()).toEqual(true);
+    });
+    it("fails fast if the openId request fails for the well-known config", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("throws if no options are available", async () => {
+      const localTransport$ = createLocalTransport$({
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: () => "",
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: async () => Promise.resolve([]),
+          // These won't be called in this error path but satisfy the type
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      });
+      await flushPromises();
+
+      expect(() => localTransport$.value).toThrow(
+        new MatrixRTCTransportMissingError(""),
+      );
+    });
+  });
 });

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
 import {
   type CallMembership,
   isLivekitTransport,
-  type LivekitTransportConfig,
   type LivekitTransport,
   isLivekitTransportConfig,
+  type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { type MatrixClient } from "matrix-js-sdk";
+import { MatrixError, type MatrixClient } from "matrix-js-sdk";
 import {
   combineLatest,
   distinctUntilChanged,
@@ -28,7 +28,10 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En
 import { type Behavior } from "../../Behavior.ts";
 import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
 import { Config } from "../../../config/Config.ts";
-import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
+import {
+  FailToGetOpenIdToken,
+  MatrixRTCTransportMissingError,
+} from "../../../utils/errors.ts";
 import {
   getSFUConfigWithOpenID,
   type OpenIDClientParts,
@@ -47,7 +50,11 @@ interface Props {
   scope: ObservableScope;
   ownMembershipIdentity: CallMembershipIdentityParts;
   memberships$: Behavior<Epoch<CallMembership[]>>;
-  client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts;
+  client: Pick<
+    MatrixClient,
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+  > &
+    OpenIDClientParts;
   roomId: string;
   useOldestMember$: Behavior<boolean>;
   useOldJwtEndpoint$: Behavior<boolean>;
@@ -141,16 +148,30 @@ export const createLocalTransport$ = ({
 const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
 /**
+ * Determine the correct Transport for the current session, including
+ * validating auth against the service to ensure it's correct.
+ * Prefers in order:
  *
- * @param client
+
- * @param roomId
+ * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
+ * 2. The transports returned via the homeserver.
+ * 3. The transports returned via .well-known.
+ * 4. The transport configured in Element Call's config.
+ *
+ * @param client The authenticated Matrix client for the current user
+ * @param roomId The ID of the room to be connected to.
+ * @param urlFromDevSettings Override URL provided by the user's local config.
  * @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token)
  * @param delayId
- * @returns
+ * @returns A fully validated transport config.
  * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
  */
 async function makeTransport(
-  client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts,
+  client: Pick<
+    MatrixClient,
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+  > &
+    OpenIDClientParts,
   membership: CallMembershipIdentityParts,
   roomId: string,
   urlFromDevSettings: string | null,
@@ -159,51 +180,127 @@ async function makeTransport(
 ): Promise<LivekitTransport & { forceOldJwtEndpoint: boolean }> {
   let transport: LivekitTransport | undefined;
   logger.trace("Searching for a preferred transport");
-  //TODO refactor this to use the jwt service returned alias.
+
-  const livekitAlias = roomId;
+  // We will call `getSFUConfigWithOpenID` once per transport here as it's our
+  // only mechanism of valiation. This means we will also ask the
+  // homeserver for a OpenID token a few times. Since OpenID tokens are single
+  // use we don't want to risk any issues by re-using a token.
+  //
+  // If the OpenID request were to fail then it's acceptable for us to fail
+  // this function early, as we assume the homeserver has got some problems.
 
   // DEVTOOL: Highest priority: Load from devtool setting
   if (urlFromDevSettings !== null) {
-    const transportFromStorage: LivekitTransport = {
+    logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
+    // Validate that the SFU is up. Otherwise, we want to fail on this
+    // as we don't permit other SFUs.
+    const config = await getSFUConfigWithOpenID(
+      client,
+      urlFromDevSettings,
+      roomId,
+    );
+    return {
       type: "livekit",
       livekit_service_url: urlFromDevSettings,
-      livekit_alias: livekitAlias,
+      livekit_alias: config.livekitAlias,
     };
-    logger.info(
-      "Using LiveKit transport from dev tools: ",
-      transportFromStorage,
-    );
-    transport = transportFromStorage;
   }
 
-  // WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
+  async function getFirstUsableTransport(
+    transports: Transport[],
+  ): Promise<LivekitTransport | null> {
+    for (const potentialTransport of transports) {
+      if (isLivekitTransportConfig(potentialTransport)) {
+        try {
+          const { livekitAlias } = await getSFUConfigWithOpenID(
+            client,
+            potentialTransport.livekit_service_url,
+            roomId,
+          );
+          return {
+            ...potentialTransport,
+            livekit_alias: livekitAlias,
+          };
+        } catch (ex) {
+          if (ex instanceof FailToGetOpenIdToken) {
+            // Explictly throw these
+            throw ex;
+          }
+          logger.debug(
+            `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
+            ex,
+          );
+        }
+      }
+    }
+    return null;
+  }
+
+  // MSC4143: Attempt to fetch transports from backend.
+  if ("_unstable_getRTCTransports" in client) {
+    try {
+      const selectedTransport = await getFirstUsableTransport(
+        await client._unstable_getRTCTransports(),
+      );
+      if (selectedTransport) {
+        logger.info("Using backend-configured SFU", selectedTransport);
+        return selectedTransport;
+      }
+    } catch (ex) {
+      if (ex instanceof MatrixError && ex.httpStatus === 404) {
+        // Expected, this is an unstable endpoint and it's not required.
+        logger.debug("Backend does not provide any RTC transports", ex);
+      } else if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      } else {
+        // We got an error that wasn't just missing support for the feature, so log it loudly.
+        logger.error(
+          "Unexpected error fetching RTC transports from backend",
+          ex,
+        );
+      }
+    }
+  }
+
+  // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
   const domain = client.getDomain();
-  if (domain && transport === undefined) {
+  if (domain) {
     // we use AutoDiscovery instead of relying on the MatrixClient having already
     // been fully configured and started
     const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
       FOCI_WK_KEY
     ];
-    if (Array.isArray(wellKnownFoci)) {
+    const selectedTransport = Array.isArray(wellKnownFoci)
-      const wellKnownTransport: LivekitTransportConfig | undefined =
+      ? await getFirstUsableTransport(wellKnownFoci)
-        wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
+      : null;
-      if (wellKnownTransport !== undefined) {
+    if (selectedTransport) {
-        logger.info("Using LiveKit transport from .well-known: ", transport);
+      logger.info("Using .well-known SFU", selectedTransport);
-        transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
+      return selectedTransport;
-      }
     }
   }
 
   // CONFIG: Least prioritized; Load from config file
   const urlFromConf = Config.get().livekit?.livekit_service_url;
-  if (urlFromConf && transport === undefined) {
+  if (urlFromConf) {
-    const transportFromConf: LivekitTransport = {
+    try {
-      type: "livekit",
+      const { livekitAlias } = await getSFUConfigWithOpenID(
-      livekit_service_url: urlFromConf,
+        client,
-      livekit_alias: livekitAlias,
+        urlFromConf,
-    };
+        roomId,
-    logger.info("Using LiveKit transport from config: ", transportFromConf);
+      );
-    transport = transportFromConf;
+      const selectedTransport: LivekitTransport = {
+        type: "livekit",
+        livekit_service_url: urlFromConf,
+        livekit_alias: livekitAlias,
+      };
+      logger.info("Using config SFU", selectedTransport);
+      return selectedTransport;
+    } catch (ex) {
+      if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      }
+      logger.error("Failed to validate config SFU", ex);
+    }
   }
 
   if (!transport) throw new MatrixRTCTransportMissingError(domain ?? "");

src/state/CallViewModel/localMember/Publisher.ts

@@ -143,7 +143,7 @@ export class Publisher {
     this.logger.debug("createAndSetupTracks called");
     const lkRoom = this.connection.livekitRoom;
     // Observe mute state changes and update LiveKit microphone/camera states accordingly
-    this.observeMuteStates(this.scope);
+    this.observeMuteStates();
 
     // Check if audio and/or video is enabled. We only create tracks if enabled,
     // because it could prompt for permission, and we don't want to do that unnecessarily.
@@ -356,10 +356,9 @@ export class Publisher {
 
   /**
    * Observe changes in the mute states and update the LiveKit room accordingly.
-   * @param scope
    * @private
    */
-  private observeMuteStates(scope: ObservableScope): void {
+  private observeMuteStates(): void {
     const lkRoom = this.connection.livekitRoom;
     this.muteStates.audio.setHandler(async (enable) => {
       try {

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -39,6 +39,7 @@ import {
   ElementCallError,
   FailToGetOpenIdToken,
 } from "../../../utils/errors.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
 import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
 
 let testScope: ObservableScope;
@@ -122,7 +123,7 @@ function setupRemoteConnection(): Connection {
       status: 200,
       body: {
         url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
@@ -259,7 +260,7 @@ describe("Start connection states", () => {
       capturedState.cause instanceof Error
     ) {
       expect(capturedState.cause.message).toContain(
-        "SFU Config fetch failed with exception Error",
+        "SFU Config fetch failed with exception",
       );
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
@@ -295,7 +296,7 @@ describe("Start connection states", () => {
         status: 200,
         body: {
           url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-          jwt: "ATOKEN",
+          jwt: testJWTToken,
         },
       };
     });
@@ -393,7 +394,7 @@ describe("remote participants", () => {
     // livekitRoom and the rtc membership in order to publish the members that are publishing
     // on this connection.
 
-    const participants: RemoteParticipant[] = [
+    let participants: RemoteParticipant[] = [
       mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
       mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
       mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
@@ -415,7 +416,22 @@ describe("remote participants", () => {
       fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
-    // All remote participants should be present
+    // At this point there should be ~~no~~ publishers
+    // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
+    // The filtering we do is just based on the matrixRTC member events.
+    expect(observedParticipants.pop()!.length).toEqual(4);
+
+    participants = [
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
+      mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
+    ];
+    participants.forEach((p) =>
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
+    );
+
+    // At this point there should be no publishers
     expect(observedParticipants.pop()!.length).toEqual(4);
   });
 

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -228,7 +228,7 @@ export class Connection {
    *
    * @param opts - Connection options {@link ConnectionOpts}.
    *
-   * @param logger
+   * @param logger - The logger to use.
    */
   public constructor(
     opts: ConnectionOpts,
@@ -238,7 +238,7 @@ export class Connection {
     this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false;
     this.logger = logger.getChild("[Connection]");
     this.logger.info(
-      `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
+      `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
     );
     const { transport, client, scope } = opts;
 

src/state/CallViewModel/remoteMembers/ConnectionFactory.ts

@@ -13,7 +13,8 @@ import {
   type BaseE2EEManager,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import E2EEWorker from "livekit-client/e2ee-worker?worker";
+// imported as inline to support worker when loaded from a cdn (cross domain)
+import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
 
@@ -45,11 +46,11 @@ export class ECConnectionFactory implements ConnectionFactory {
    * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
    * @param devices - Used for video/audio out/in capture options.
    * @param processorState$ - Effects like background blur (only for publishing connection?)
-   * @param livekitKeyProvider
+   * @param livekitKeyProvider - Optional key provider for end-to-end encryption.
    * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
+   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    * @param echoCancellation - Whether to enable echo cancellation for audio capture.
    * @param noiseSuppression - Whether to enable noise suppression for audio capture.
-   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    */
   public constructor(
     private client: OpenIDClientParts,

src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts

@@ -293,47 +293,47 @@ describe("connectionManagerData$ stream", () => {
         a: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
           return true;
         }),
         b: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
+          expect(
-            "user1A",
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
-          );
+          ).toBe("user1A");
           return true;
         }),
         c: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
+          expect(
-            "user1A",
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
-          );
+          ).toBe("user1A");
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
+          expect(
-            "user2A",
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
-          );
+          ).toBe("user2A");
           return true;
         }),
         d: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
+          expect(
-            "user1A",
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
-          );
+          ).toBe("user1A");
-          expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
+          expect(
-            "user1B",
+            data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
-          );
+          ).toBe("user1B");
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
+          expect(
-            "user2A",
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
-          );
+          ).toBe("user2A");
           return true;
         }),
       });

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -20,8 +20,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 
 export class ConnectionManagerData {
-  private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
+  private readonly store: Map<
-    new Map();
+    string,
+    { connection: Connection; participants: RemoteParticipant[] }
+  > = new Map();
 
   public constructor() {}
 
@@ -29,9 +31,9 @@ export class ConnectionManagerData {
     const key = this.getKey(connection.transport);
     const existing = this.store.get(key);
     if (!existing) {
-      this.store.set(key, [connection, participants]);
+      this.store.set(key, { connection, participants });
     } else {
-      existing[1].push(...participants);
+      existing.participants.push(...participants);
     }
   }
 
@@ -40,20 +42,24 @@ export class ConnectionManagerData {
   }
 
   public getConnections(): Connection[] {
-    return Array.from(this.store.values()).map(([connection]) => connection);
+    return Array.from(this.store.values()).map(({ connection }) => connection);
   }
 
   public getConnectionForTransport(
     transport: LivekitTransport,
   ): Connection | null {
-    return this.store.get(this.getKey(transport))?.[0] ?? null;
+    return this.store.get(this.getKey(transport))?.connection ?? null;
   }
 
-  public getParticipantForTransport(
+  public getParticipantsForTransport(
     transport: LivekitTransport,
   ): RemoteParticipant[] {
     const key = transport.livekit_service_url + "|" + transport.livekit_alias;
-    return this.store.get(key)?.[1] ?? [];
+    const existing = this.store.get(key);
+    if (existing) {
+      return existing.participants;
+    }
+    return [];
   }
 }
 
@@ -74,9 +80,11 @@ export interface IConnectionManager {
 
 /**
  * Crete a `ConnectionManager`
- * @param scope the observable scope used by this object.
+ * @param props - Configuration object
- * @param connectionFactory used to create new connections.
+ * @param props.scope - The observable scope used by this object
- * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props.connectionFactory - Used to create new connections
+ * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props.logger - The logger to use
  *   Each of these behaviors can be interpreted as subscribed list of transports.
  *
  *   Using `registerTransports` independent external modules can control what connections
@@ -207,6 +215,7 @@ export function createConnectionManager$({
         );
 
         // probably not required
+
         if (listOfConnectionsWithRemoteParticipants.length === 0) {
           return of(new Epoch(new ConnectionManagerData(), epoch));
         }

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -249,7 +249,7 @@ describe("Publication edge case", () => {
       constant(connectionWithPublisher),
     );
 
-    const matrixLivekitMember$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -257,7 +257,7 @@ describe("Publication edge case", () => {
       } as unknown as IConnectionManager,
     });
     await flushPromises();
-    expect(matrixLivekitMember$.value.value).toSatisfy(
+    expect(matrixLivekitMembers$.value.value).toSatisfy(
       (data: RemoteMatrixLivekitMember[]) => {
         expect(data.length).toEqual(2);
         expect(data[0].membership$.value).toBe(bobMembership);

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -100,7 +100,7 @@ export function createMatrixLivekitMembers$({
         function* ([membershipsWithTransport, managerData]) {
           for (const { membership, transport } of membershipsWithTransport) {
             const participants = transport
-              ? managerData.getParticipantForTransport(transport)
+              ? managerData.getParticipantsForTransport(transport)
               : [];
             const participant =
               participants.find(

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -13,7 +13,11 @@ import fetchMock from "fetch-mock";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-import { type Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts";
+import {
+  type Epoch,
+  ObservableScope,
+  trackEpoch,
+} from "../../ObservableScope.ts";
 import { ECConnectionFactory } from "./ConnectionFactory.ts";
 import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
 import {
@@ -31,6 +35,7 @@ import {
 import { createConnectionManager$ } from "./ConnectionManager.ts";
 import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
 import { constant } from "../../Behavior.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
 
 // Test the integration of ConnectionManager and MatrixLivekitMerger
 
@@ -83,7 +88,7 @@ beforeEach(() => {
       status: 200,
       body: {
         url: `wss://${domain}/livekit/sfu`,
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
@@ -124,14 +129,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
       ownMembershipIdentity: ownMemberMock,
     });
 
-    const matrixLivekitItems$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$:
         membershipsAndTransports.membershipsWithTransport$,
       connectionManager,
     });
 
-    expectObservable(matrixLivekitItems$).toBe(vMarble, {
+    expectObservable(matrixLivekitMembers$).toBe(vMarble, {
       a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(1);

src/useAudioContext.tsx

@@ -22,9 +22,12 @@ import * as controls from "./controls";
  * Play a sound though a given AudioContext. Will take
  * care of connecting the correct buffer and gating
  * through gain.
- * @param volume The volume to play at.
  * @param ctx The context to play through.
  * @param buffer The buffer to play.
+ * @param volume The volume to play at.
+ * @param stereoPan The stereo pan to apply.
+ * @param delayS Delay in seconds before starting playing.
+ * @param abort Optional AbortController that can be used to stop playback.
  * @returns A promise that resolves when the sound has finished playing.
  */
 async function playSound(
@@ -55,9 +58,11 @@ async function playSound(
  * Play a sound though a given AudioContext, looping until stopped. Will take
  * care of connecting the correct buffer and gating
  * through gain.
- * @param volume The volume to play at.
  * @param ctx The context to play through.
  * @param buffer The buffer to play.
+ * @param volume The volume to play at.
+ * @param stereoPan The stereo pan to apply.
+ * @param delayS Delay in seconds between each loop.
  * @returns A function used to end the sound. This function will return a promise when the sound has stopped.
  */
 function playSoundLooping(
@@ -120,7 +125,7 @@ interface UseAudioContext<S extends string> {
 /**
  * Add an audio context which can be used to play
  * a set of preloaded sounds.
- * @param props
+ * @param props The properties for the audio context.
  * @returns Either an instance that can be used to play sounds, or null if not ready.
  */
 export function useAudioContext<S extends string>(

src/utils/displayname.ts

@@ -77,6 +77,13 @@ export function shouldDisambiguate(
   );
 }
 
+/**
+ * Calculates a display name for a member, optionally disambiguating it.
+ * @param member - The member to calculate the display name for.
+ * @param member.rawDisplayName - The raw display name of the member
+ * @param member.userId - The user ID of the member
+ * @param disambiguate - Whether to disambiguate the display name.
+ */
 export function calculateDisplayName(
   member: { rawDisplayName?: string; userId: string },
   disambiguate: boolean,

src/utils/errors.ts

@@ -57,9 +57,16 @@ export class ElementCallError extends Error {
   }
 }
 
+/**
+ * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
+ */
 export class MatrixRTCTransportMissingError extends ElementCallError {
   public domain: string;
 
+  /**
+   * Creates an instance of MatrixRTCTransportMissingError.
+   * @param domain - The domain where the MatrixRTC transport is missing.
+   */
   public constructor(domain: string) {
     super(
       t("error.call_is_not_supported"),
@@ -75,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that the connection to the call was lost and could not be re-established.
+ */
 export class ConnectionLostError extends ElementCallError {
   public constructor() {
     super(
@@ -86,7 +96,16 @@ export class ConnectionLostError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure in the membership manager causing the join call
+ * operation to fail.
+ */
 export class MembershipManagerError extends ElementCallError {
+  /**
+   * Creates an instance of MembershipManagerError.
+   *
+   * @param error - The underlying error that caused the membership manager failure.
+   */
   public constructor(error: Error) {
     super(
       t("error.membership_manager"),
@@ -98,6 +117,9 @@ export class MembershipManagerError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that end-to-end encryption is not supported in the current environment.
+ */
 export class E2EENotSupportedError extends ElementCallError {
   public constructor() {
     super(
@@ -109,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating an unknown issue occurred during a call operation.
+ */
 export class UnknownCallError extends ElementCallError {
+  /**
+   * Creates an instance of UnknownCallError.
+   * @param error - The underlying error that caused the unknown issue.
+   */
   public constructor(error: Error) {
     super(
       t("error.generic"),
@@ -122,7 +151,14 @@ export class UnknownCallError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure to obtain an OpenID token.
+ */
 export class FailToGetOpenIdToken extends ElementCallError {
+  /**
+   * Creates an instance of FailToGetOpenIdToken.
+   * @param error - The underlying error that caused the failure.
+   */
   public constructor(error: Error) {
     super(
       t("error.generic"),
@@ -135,7 +171,14 @@ export class FailToGetOpenIdToken extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure to start publishing on a LiveKit connection.
+ */
 export class FailToStartLivekitConnection extends ElementCallError {
+  /**
+   * Creates an instance of FailToStartLivekitConnection.
+   * @param e - An optional error message providing additional context.
+   */
   public constructor(e?: string) {
     super(
       t("error.failed_to_start_livekit"),
@@ -146,6 +189,9 @@ export class FailToStartLivekitConnection extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that a LiveKit's server has hit its track limits.
+ */
 export class InsufficientCapacityError extends ElementCallError {
   public constructor() {
     super(
@@ -157,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that room creation is restricted by the SFU.
+ * Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
+ */
 export class SFURoomCreationRestrictedError extends ElementCallError {
   public constructor() {
     super(

src/utils/matrix.ts

@@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string {
  * Applies some basic sanitisation to a room name that the user
  * has given us
  * @param input The room name from the user
- * @param client A matrix client object
  */
 export function sanitiseRoomNameInput(input: string): string {
   // check to see if the user has entered a fully qualified room
@@ -304,8 +303,9 @@ export async function createRoom(
 /**
  * Returns an absolute URL to that will load Element Call with the given room
  * @param roomId ID of the room
- * @param roomName Name of the room
  * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
+ * @param roomName Name of the room
+ * @param viaServers Optional list of servers to include as 'via' parameters in the URL
  */
 export function getAbsoluteRoomUrl(
   roomId: string,
@@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl(
 /**
  * Returns a relative URL to that will load Element Call with the given room
  * @param roomId ID of the room
- * @param roomName Name of the room
  * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
+ * @param roomName Name of the room
+ * @param viaServers Optional list of servers to include as 'via' parameters in the URL
  */
 export function getRelativeRoomUrl(
   roomId: string,

src/utils/media.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 /**
  * Finds a media device with label matching 'deviceName'
  * @param deviceName The label of the device to look for
+ * @param kind The kind of media device to look for
  * @param devices The list of devices to search
  * @returns A matching media device or undefined if no matching device was found
  */

src/utils/observable.ts

@@ -135,7 +135,6 @@ interface ItemHandle<Data, Item> {
  * requested at a later time, and destroyed (have their scope ended) when the
  * key is no longer requested.
  *
- * @param input$ The input value to be mapped.
  * @param generator A generator function yielding a tuple of keys and the
  *   currently associated data for each item that it wants to exist.
  * @param factory A function constructing an individual item, given the item's key,

src/utils/test-fixtures.ts

@@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD");
 export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
   rawDisplayName: "\u202eevaD",
 });
+
+export const testJWTToken = [
+  {}, // header
+  {
+    // payload
+    sub: "@me:example.org:ABCDEF",
+    video: {
+      room: "!example_room_id",
+    },
+  },
+  {}, // signature
+]
+  .map((d) => global.btoa(JSON.stringify(d)))
+  .join(".");

src/widget.ts

@@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => {
   try {
     const { widgetId, parentUrl } = getUrlParams();
 
+    const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } =
+      getUrlParams();
+    if (!roomId) throw new Error("Room ID must be supplied");
+    if (!userId) throw new Error("User ID must be supplied");
+    if (!deviceId) throw new Error("Device ID must be supplied");
+    if (!baseUrl) throw new Error("Base URL must be supplied");
     if (widgetId && parentUrl) {
       const parentOrigin = new URL(parentUrl).origin;
       logger.info("Widget API is available");
@@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => {
       // We need to do this now rather than later because it has capabilities to
       // request, and is responsible for starting the transport (should it be?)
 
-      const {
-        roomId,
-        userId,
-        deviceId,
-        baseUrl,
-        e2eEnabled,
-        allowIceFallback,
-      } = getUrlParams();
-      if (!roomId) throw new Error("Room ID must be supplied");
-      if (!userId) throw new Error("User ID must be supplied");
-      if (!deviceId) throw new Error("Device ID must be supplied");
-      if (!baseUrl) throw new Error("Base URL must be supplied");
-
       // These are all the event types the app uses
       const sendEvent = [
         EventType.CallNotify, // Sent as a deprecated fallback

tsconfig.json

@@ -50,6 +50,11 @@
 
     "plugins": [{ "name": "typescript-eslint-language-service" }]
   },
-  "include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"],
+  "include": [
+    "./src/**/*.ts",
+    "./src/**/*.tsx",
+    "./playwright/**/*.ts",
+    "./sdk/**/*.ts"
+  ],
   "exclude": ["**.test.ts"]
 }

vite-sdk.config.ts

@@ -0,0 +1,28 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { defineConfig, mergeConfig } from "vite";
+import nodePolyfills from "vite-plugin-node-stdlib-browser";
+
+const base = "./";
+
+// Config for embedded deployments (possibly hosted under a non-root path)
+export default defineConfig(() => ({
+  worker: { format: "es" as const },
+  base, // Use relative URLs to allow the app to be hosted under any path
+  build: {
+    sourcemap: true,
+    manifest: true,
+    lib: {
+      formats: ["es" as const],
+      entry: "./sdk/main.ts",
+      name: "MatrixrtcSdk",
+      fileName: "matrixrtc-sdk",
+    },
+  },
+  plugins: [nodePolyfills()],
+}));

vite.config.ts

@@ -7,14 +7,17 @@ Please see LICENSE in the repository root for full details.
 
 import {
   loadEnv,
+  PluginOption,
   searchForWorkspaceRoot,
   type ConfigEnv,
   type UserConfig,
 } from "vite";
 import svgrPlugin from "vite-plugin-svgr";
 import { createHtmlPlugin } from "vite-plugin-html";
+
 import { codecovVitePlugin } from "@codecov/vite-plugin";
 import { sentryVitePlugin } from "@sentry/vite-plugin";
+
 import react from "@vitejs/plugin-react";
 import { realpathSync } from "fs";
 import * as fs from "node:fs";
@@ -31,7 +34,7 @@ export default ({
   // In future we might be able to do what is needed via code splitting at
   // build time.
   process.env.VITE_PACKAGE = packageType ?? "full";
-  const plugins = [
+  const plugins: PluginOption[] = [
     react(),
     svgrPlugin({
       svgrOptions: {
@@ -41,16 +44,6 @@ export default ({
       },
     }),
 
-    createHtmlPlugin({
-      entry: "src/main.tsx",
-      inject: {
-        data: {
-          brand: env.VITE_PRODUCT_NAME || "Element Call",
-          packageType: process.env.VITE_PACKAGE,
-        },
-      },
-    }),
-
     codecovVitePlugin({
       enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
       bundleName: "element-call",
@@ -73,6 +66,18 @@ export default ({
     );
   }
 
+  plugins.push(
+    createHtmlPlugin({
+      entry: "src/main.tsx",
+      inject: {
+        data: {
+          brand: env.VITE_PRODUCT_NAME || "Element Call",
+          packageType: process.env.VITE_PACKAGE,
+        },
+      },
+    }),
+  );
+
   // The crypto WASM module is imported dynamically. Since it's common
   // for developers to use a linked copy of matrix-js-sdk or Rust
   // crypto (which could reside anywhere on their file system), Vite