Merge pull request #3701 from element-hq/toger5/sdk-improvements

[WIP] SDK target improvements

.github/workflows/build-element-call.yaml

@@ -7,7 +7,7 @@ on:
         type: string
       package:
         type: string # This would ideally be a `choice` type, but that isn't supported yet
-        description: The package type to be built. Must be one of 'full' or 'embedded'
+        description: The package type to be built. Must be one of 'full', 'embedded', or 'sdk'
         required: true
       build_mode:
         type: string # This would ideally be a `choice` type, but that isn't supported yet

.github/workflows/build.yaml

@@ -69,3 +69,17 @@ jobs:
       SENTRY_URL: ${{ secrets.SENTRY_URL }}
       SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  build_sdk_element_call:
+    # Use the embedded package vite build
+    uses: ./.github/workflows/build-element-call.yaml
+    with:
+      package: sdk
+      vite_app_version: ${{ github.event.release.tag_name || github.sha }}
+      build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
+    secrets:
+      SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+      SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      SENTRY_URL: ${{ secrets.SENTRY_URL }}
+      SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/deploy-to-netlify.yaml

@@ -14,6 +14,10 @@ on:
       deployment_ref:
         required: true
         type: string
+      package:
+        required: true
+        type: string
+        description: Which package to deploy - 'full', 'embedded', or 'sdk'
       artifact_run_id:
         required: false
         type: string
@@ -50,7 +54,7 @@ jobs:
         with:
           github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
           run-id: ${{ inputs.artifact_run_id }}
-          name: build-output-full
+          name: build-output-${{ inputs.package }}
           path: webapp
 
       - name: Add redirects file
@@ -58,15 +62,17 @@ jobs:
         run: curl -s https://raw.githubusercontent.com/element-hq/element-call/main/config/netlify_redirects > webapp/_redirects
 
       - name: Add config file
-        run: curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview.json" > webapp/config.json
+        run: |
-
+          if [ "${{ inputs.package }}" = "full" ]; then
+            curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview_sdk.json" > webapp/config.json
+          fi
       - name: ☁️ Deploy to Netlify
         id: netlify
         uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0
         with:
           publish-dir: webapp
           deploy-message: "Deploy from GitHub Actions"
-          alias: pr${{ inputs.pr_number }}
+          alias: ${{ inputs.package == 'sdk' && format('pr{0}-sdk', inputs.pr_number) || format('pr{0}', inputs.pr_number) }}
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}

.github/workflows/pr-deploy.yaml

@@ -20,7 +20,7 @@ jobs:
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}
           branch: ${{ github.event.workflow_run.head_branch }}
 
-  netlify:
+  netlify-full:
     needs: prdetails
     permissions:
       deployments: write
@@ -31,6 +31,24 @@ jobs:
       pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
       pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
       deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
+      package: full
+    secrets:
+      ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
+      NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+      NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+
+  netlify-sdk:
+    needs: prdetails
+    permissions:
+      deployments: write
+    uses: ./.github/workflows/deploy-to-netlify.yaml
+    with:
+      artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }}
+      pr_number: ${{ needs.prdetails.outputs.pr_number }}
+      pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
+      pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
+      deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
+      package: sdk
     secrets:
       ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
       NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}

package.json

@@ -13,8 +13,9 @@
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
     "build:embedded:production": "yarn build:embedded",
     "build:embedded:development": "yarn build:embedded --mode development",
-    "build:sdk": "yarn build:full --config vite-sdk.config.js",
     "build:sdk:development": "yarn build:sdk --mode development",
+    "build:sdk": "yarn build:full --config vite-sdk.config.js",
+    "build:sdk:production": "yarn build:sdk",
     "serve": "vite preview",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",
@@ -104,7 +105,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop",
+    "matrix-js-sdk": "40.2.0-rc.0",
     "matrix-widget-api": "^1.16.1",
     "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",

sdk/README.md

@@ -21,7 +21,7 @@ in the repository root.
 
 It will create a `dist` folder containing the compiled js file.
 
-This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server.
+This file needs to be hosted. Locally (via `npx serve -l 1234 --cors`) or on a remote server.
 
 Now you just need to add the widget to element web via:
 

sdk/helper.ts

@@ -12,15 +12,12 @@ Please see LICENSE in the repository root for full details.
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import { scan } from "rxjs";
 
-import { widget as _widget } from "../src/widget";
+import { type WidgetHelpers } from "../src/widget";
 import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
 
 export const logger = rootLogger.getChild("[MatrixRTCSdk]");
 
-if (!_widget) throw Error("No widget. This webapp can only start as a widget");
+export const tryMakeSticky = (widget: WidgetHelpers): void => {
-export const widget = _widget;
-
-export const tryMakeSticky = (): void => {
   logger.info("try making sticky MatrixRTCSdk");
   void widget.api
     .setAlwaysOnScreen(true)

sdk/main.ts

@@ -30,8 +30,8 @@ import {
 } from "rxjs";
 import {
   type CallMembership,
-  MatrixRTCSession,
   MatrixRTCSessionEvent,
+  MatrixRTCSessionManager,
 } from "matrix-js-sdk/lib/matrixrtc";
 import {
   type Room as LivekitRoom,
@@ -50,14 +50,12 @@ import { getUrlParams } from "../src/UrlParams";
 import { MuteStates } from "../src/state/MuteStates";
 import { MediaDevices } from "../src/state/MediaDevices";
 import { E2eeType } from "../src/e2ee/e2eeType";
+import { currentAndPrev, logger, TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
 import {
-  currentAndPrev,
+  ElementWidgetActions,
-  logger,
+  widget as _widget,
-  TEXT_LK_TOPIC,
+  initializeWidget,
-  tryMakeSticky,
+} from "../src/widget";
-  widget,
-} from "./helper";
-import { ElementWidgetActions, initializeWidget } from "../src/widget";
 import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
 
 interface MatrixRTCSdk {
@@ -68,7 +66,7 @@ interface MatrixRTCSdk {
   join: () => void;
   /** @throws on leave errors */
   leave: () => void;
-  data$: Observable<{ sender: string; data: string }>;
+  data$: Observable<{ rtcBackendIdentity: string; data: string }>;
   /**
    * flattened list of members
    */
@@ -79,32 +77,54 @@ interface MatrixRTCSdk {
       participant: LocalParticipant | RemoteParticipant | null;
     }[]
   >;
+  /**
+   * flattened local members
+   */
+  localMember$: Behavior<{
+    connection: Connection | null;
+    membership: CallMembership;
+    participant: LocalParticipant | null;
+  } | null>;
   /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
   connected$: Behavior<boolean>;
   sendData?: (data: unknown) => Promise<void>;
+  sendRoomMessage?: (message: string) => Promise<void>;
 }
 
 export async function createMatrixRTCSdk(
   application: string = "m.call",
   id: string = "",
+  sticky: boolean = false,
 ): Promise<MatrixRTCSdk> {
-  initializeWidget();
+  const scope = new ObservableScope();
+
+  // widget client
+  initializeWidget(application, true);
+  const widget = _widget;
+  if (!widget) throw Error("No widget. This webapp can only start as a widget");
   const client = await widget.client;
   logger.info("client created");
-  const scope = new ObservableScope();
+
+  // url params
   const { roomId } = getUrlParams();
   if (roomId === null) throw Error("could not get roomId from url params");
-
   const room = client.getRoom(roomId);
   if (room === null) throw Error("could not get room from client");
 
+  // rtc session
+  const slot = { application, id };
+  const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot);
+  rtcSessionManager.start();
+  const rtcSession = rtcSessionManager.getRoomSession(room);
+
+  // media devices
   const mediaDevices = new MediaDevices(scope);
   const muteStates = new MuteStates(scope, mediaDevices, {
-    audioEnabled: true,
+    audioEnabled: false,
-    videoEnabled: true,
+    videoEnabled: false,
   });
-  const slot = { application, id };
+
-  const rtcSession = new MatrixRTCSession(client, room, slot);
+  // call view model
   const callViewModel = createCallViewModel$(
     scope,
     rtcSession,
@@ -117,8 +137,9 @@ export async function createMatrixRTCSdk(
     constant({ supported: false, processor: undefined }),
   );
   logger.info("CallViewModelCreated");
+
   // create data listener
-  const data$ = new Subject<{ sender: string; data: string }>();
+  const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>();
 
   const lkTextStreamHandlerFunction = async (
     reader: TextStreamReader,
@@ -140,7 +161,7 @@ export async function createMatrixRTCSdk(
     if (participants && participants.includes(participantInfo.identity)) {
       const text = await reader.readAll();
       logger.info(`Received text: ${text}`);
-      data$.next({ sender: participantInfo.identity, data: text });
+      data$.next({ rtcBackendIdentity: participantInfo.identity, data: text });
     } else {
       logger.warn(
         "Received text from unknown participant",
@@ -230,6 +251,16 @@ export async function createMatrixRTCSdk(
     }
   };
 
+  const sendRoomMessage = async (message: string): Promise<void> => {
+    const messageString = JSON.stringify(message);
+    logger.info("try sending to room: ", messageString);
+    try {
+      await client.sendTextMessage(room.roomId, message);
+    } catch (e) {
+      logger.error("failed sending to room: ", messageString, e);
+    }
+  };
+
   // after hangup gets called
   const leaveSubs = callViewModel.leave$.subscribe(() => {
     const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
@@ -267,7 +298,7 @@ export async function createMatrixRTCSdk(
   return {
     join: (): void => {
       // first lets try making the widget sticky
-      tryMakeSticky();
+      if (sticky) tryMakeSticky(widget);
       callViewModel.join();
     },
     leave: (): void => {
@@ -276,6 +307,28 @@ export async function createMatrixRTCSdk(
       livekitRoomItemsSub.unsubscribe();
     },
     data$,
+    localMember$: scope.behavior(
+      callViewModel.localMatrixLivekitMember$.pipe(
+        tap((member) =>
+          logger.info("localMatrixLivekitMember$ next: ", member),
+        ),
+        switchMap((member) => {
+          if (member === null) return of(null);
+          return combineLatest([
+            member.connection$,
+            member.membership$,
+            member.participant.value$,
+          ]).pipe(
+            map(([connection, membership, participant]) => ({
+              connection,
+              membership,
+              participant,
+            })),
+          );
+        }),
+        tap((member) => logger.info("localMember$ next: ", member)),
+      ),
+    ),
     connected$: callViewModel.connected$,
     members$: scope.behavior(
       callViewModel.matrixLivekitMembers$.pipe(
@@ -302,5 +355,6 @@ export async function createMatrixRTCSdk(
       [],
     ),
     sendData,
+    sendRoomMessage,
   };
 }

src/livekit/openIDSFU.ts

@@ -146,7 +146,7 @@ export async function getSFUConfigWithOpenID(
       } else {
         logger?.warn(
           `Failed fetching jwt with matrix 2.0 endpoint other issues ->`,
-          `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`,
+          `(not going to try with legacy endpoint if forceMatrix2Jwt is set to false (it is ${forceMatrix2Jwt}), we did not get a not supported error from the sfu)`,
           e,
         );
         // Make this throw a hard error in case we force the matrix2.0 endpoint.

src/widget.test.ts

@@ -6,12 +6,11 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { beforeAll, describe, expect, vi, it } from "vitest";
-import { createRoomWidgetClient, EventType } from "matrix-js-sdk";
+import { createRoomWidgetClient } from "matrix-js-sdk";
 
 import { getUrlParams } from "./UrlParams";
 import { initializeWidget, widget } from "./widget";
 import { Config } from "./config/Config";
-import { ElementCallReactionEventType } from "./reactions";
 
 vi.mock("matrix-js-sdk", { spy: true });
 const createRoomWidgetClientSpy = vi.mocked(createRoomWidgetClient);
@@ -35,7 +34,7 @@ vi.mock("./UrlParams", () => ({
   })),
 }));
 
-initializeWidget();
+initializeWidget("ANYRTCAPP");
 describe("widget", () => {
   beforeAll(() => {});
 
@@ -52,48 +51,39 @@ describe("widget", () => {
     expect(widget).toBeDefined();
     expect(configInitSpy).toHaveBeenCalled();
     const sendEvent = [
-      EventType.CallNotify, // Sent as a deprecated fallback
+      "org.matrix.msc4075.call.notify", // Sent as a deprecated fallback
-      EventType.RTCNotification,
+      "org.matrix.msc4075.rtc.notification",
     ];
     const sendRecvEvent = [
       "org.matrix.rageshake_request",
-      EventType.CallEncryptionKeysPrefix,
+      "io.element.call.encryption_keys",
-      EventType.Reaction,
+      "m.reaction",
-      EventType.RoomRedaction,
+      "m.room.redaction",
-      ElementCallReactionEventType,
+      "io.element.call.reaction",
-      EventType.RTCDecline,
+      "org.matrix.msc4310.rtc.decline",
-      EventType.RTCMembership,
+      "org.matrix.msc4143.rtc.member",
     ];
 
     const sendState = [
-      "myYser", // Legacy call membership events
+      { eventType: "org.matrix.msc3401.call.member", stateKey: "myYser" }, // Legacy call membership events
-      `_myYser_AAAAA_m.call`, // Session membership events
+      {
-      `myYser_AAAAA_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it
+        eventType: "org.matrix.msc3401.call.member",
-    ].map((stateKey) => ({
+        stateKey: `_myYser_AAAAA_ANYRTCAPP`,
-      eventType: EventType.GroupCallMemberPrefix,
+      }, // Session membership events
-      stateKey,
+      {
-    }));
+        eventType: "org.matrix.msc3401.call.member",
+        stateKey: `myYser_AAAAA_ANYRTCAPP`,
+      }, // The above with no leading underscore, for room versions whose auth rules allow it
+    ];
     const receiveState = [
-      { eventType: EventType.RoomCreate },
+      { eventType: "m.room.create" },
-      { eventType: EventType.RoomName },
+      { eventType: "m.room.name" },
-      { eventType: EventType.RoomMember },
+      { eventType: "m.room.member" },
-      { eventType: EventType.RoomEncryption },
+      { eventType: "m.room.encryption" },
-      { eventType: EventType.GroupCallMemberPrefix },
+      { eventType: "org.matrix.msc3401.call.member" },
     ];
 
-    const sendRecvToDevice = [
+    const sendRecvToDevice = ["io.element.call.encryption_keys"];
-      EventType.CallInvite,
-      EventType.CallCandidates,
-      EventType.CallAnswer,
-      EventType.CallHangup,
-      EventType.CallReject,
-      EventType.CallSelectAnswer,
-      EventType.CallNegotiate,
-      EventType.CallSDPStreamMetadataChanged,
-      EventType.CallSDPStreamMetadataChangedPrefix,
-      EventType.CallReplaces,
-      EventType.CallEncryptionKeysPrefix,
-    ];
 
     expect(createRoomWidgetClientSpy.mock.calls[0][1]).toStrictEqual({
       sendEvent: [...sendEvent, ...sendRecvEvent],

src/widget.ts

@@ -68,7 +68,10 @@ export let widget: WidgetHelpers | null;
  */
 // this needs to be a seperate call and cannot be done on import to allow us to spy on methods in here before
 // execution.
-export const initializeWidget = (): void => {
+export const initializeWidget = (
+  rtcApplication: string = "m.call",
+  sendRoomEvents = false,
+): void => {
   try {
     const {
       widgetId,
@@ -116,6 +119,9 @@ export const initializeWidget = (): void => {
         EventType.CallNotify, // Sent as a deprecated fallback
         EventType.RTCNotification,
       ];
+      if (sendRoomEvents) {
+        sendEvent.push(EventType.RoomMessage);
+      }
       const sendRecvEvent = [
         "org.matrix.rageshake_request",
         EventType.CallEncryptionKeysPrefix,
@@ -128,8 +134,8 @@ export const initializeWidget = (): void => {
 
       const sendState = [
         userId, // Legacy call membership events
-        `_${userId}_${deviceId}_m.call`, // Session membership events
+        `_${userId}_${deviceId}_${rtcApplication}`, // Session membership events
-        `${userId}_${deviceId}_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it
+        `${userId}_${deviceId}_${rtcApplication}`, // The above with no leading underscore, for room versions whose auth rules allow it
       ].map((stateKey) => ({
         eventType: EventType.GroupCallMemberPrefix,
         stateKey,
@@ -142,19 +148,7 @@ export const initializeWidget = (): void => {
         { eventType: EventType.GroupCallMemberPrefix },
       ];
 
-      const sendRecvToDevice = [
+      const sendRecvToDevice = [EventType.CallEncryptionKeysPrefix];
-        EventType.CallInvite,
-        EventType.CallCandidates,
-        EventType.CallAnswer,
-        EventType.CallHangup,
-        EventType.CallReject,
-        EventType.CallSelectAnswer,
-        EventType.CallNegotiate,
-        EventType.CallSDPStreamMetadataChanged,
-        EventType.CallSDPStreamMetadataChangedPrefix,
-        EventType.CallReplaces,
-        EventType.CallEncryptionKeysPrefix,
-      ];
 
       const client = createRoomWidgetClient(
         api,

yarn.lock

@@ -8364,7 +8364,7 @@ __metadata:
     livekit-client: "npm:^2.13.0"
     lodash-es: "npm:^4.17.21"
     loglevel: "npm:^1.9.1"
-    matrix-js-sdk: "matrix-org/matrix-js-sdk#develop"
+    matrix-js-sdk: "npm:40.2.0-rc.0"
     matrix-widget-api: "npm:^1.16.1"
     node-stdlib-browser: "npm:^1.3.1"
     normalize.css: "npm:^8.0.1"
@@ -11452,9 +11452,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@matrix-org/matrix-js-sdk#develop":
+"matrix-js-sdk@npm:40.2.0-rc.0":
-  version: 40.1.0
+  version: 40.2.0-rc.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f2157f28bbadf2898fe21991f69ccb2af40df326"
+  resolution: "matrix-js-sdk@npm:40.2.0-rc.0"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0"
@@ -11470,7 +11470,7 @@ __metadata:
     sdp-transform: "npm:^3.0.0"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:13"
-  checksum: 10c0/d646b9214abbf0b9126760105edd9c57be7ffe8b53ae4acd5fefe841a51ad7d78fa57130922b3eac65ff2266b43f31ea60b4bdda9481e6bf8f1808d96726ed8a
+  checksum: 10c0/82311a60bc0fd2c8f5dff5219d05744d45577c2ea3145d17bef71e6ea194f4bb16f4557a5e74839dbc1b17fe95e08f0f510b7fd0da10f82dda8cb55ce28cd5f5
   languageName: node
   linkType: hard