Introduce MatrixMemberMetadata and use it to simplify username and

avatar computation This removes member from the tiles entirely!
2025-11-11 15:51:48 +01:00
parent 93c4dc5beb
commit 85f659bcc9
9 changed files with 256 additions and 322 deletions
--- a/src/state/CallViewModel/CallViewModel.ts
+++ b/src/state/CallViewModel/CallViewModel.ts
@@ -109,7 +109,10 @@ import {
  createReceivedDecline$,
  createSentCallNotification$,
 } from "./CallNotificationLifecycle.ts";
-import { createRoomMembers$ } from "./remoteMembers/displayname.ts";
+import {
+  createMatrixMemberMetadata$,
+  createRoomMembers$,
+} from "./remoteMembers/MatrixMemberMetadata.ts";

 const logger = rootLogger.getChild("[CallViewModel]");
 //TODO
@@ -240,7 +243,6 @@ export class CallViewModel {
    membershipsWithTransport$:
      this.membershipsAndTransports.membershipsWithTransport$,
    connectionManager: this.connectionManager,
-    matrixRoom: this.matrixRoom,
  });

  private connectOptions$ = this.scope.behavior(
@@ -280,11 +282,9 @@ export class CallViewModel {
    options: this.options,
    localUser: { userId: this.userId, deviceId: this.deviceId },
  });
-
+  public autoLeave$ = this.callLifecycle.autoLeave$;
  // ------------------------------------------------------------------------
-  // ROOM MEMBER tracking TODO
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom);
+
  /**
   * If there is a configuration error with the call (e.g. misconfigured E2EE).
   * This is a fatal error that prevents the call from being created/joined.
@@ -305,14 +305,6 @@ export class CallViewModel {
    "user" | "timeout" | "decline" | "allOthersLeft"
  >();

-  /**
-   * Whether we are joined to the call. This reflects our local state rather
-   * than whether all connections are truly up and running.
-   */
-  // DISCUSS ? lets think why we need joined and how to do it better
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  private readonly joined$ = this.localMembership.connected$;
-
  /**
   * Whether various media/event sources should pretend to be disconnected from
   * all network input, even if their connection still technically works.
@@ -385,6 +377,14 @@ export class CallViewModel {
    ),
  );

+  private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom);
+
+  private matrixMemberMetadataStore = createMatrixMemberMetadata$(
+    this.scope,
+    this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))),
+    this.roomMembers$,
+  );
+
  /**
   * List of user media (camera feeds) that we want tiles for.
   */
@@ -400,20 +400,10 @@ export class CallViewModel {
            userId,
            participant$,
            connection$,
-            displayName$,
-            mxcAvatarUrl$,
          } of matrixLivekitMembers)
            for (let dup = 0; dup < 1 + duplicateTiles; dup++)
              yield {
-                keys: [
-                  dup,
-                  participantId,
-                  userId,
-                  participant$,
-                  connection$,
-                  displayName$,
-                  mxcAvatarUrl$,
-                ],
+                keys: [dup, participantId, userId, participant$, connection$],
                data: undefined,
              };
        },
@@ -425,8 +415,6 @@ export class CallViewModel {
          userId,
          participant$,
          connection$,
-          displayName$,
-          mxcAvatarUrl$,
        ) => {
          const livekitRoom$ = scope.behavior(
            connection$.pipe(map((c) => c?.livekitRoom)),
@@ -434,6 +422,11 @@ export class CallViewModel {
          const focusUrl$ = scope.behavior(
            connection$.pipe(map((c) => c?.transport.livekit_service_url)),
          );
+          const displayName$ = scope.behavior(
+            this.matrixMemberMetadataStore
+              .createDisplayNameBehavior$(userId)
+              .pipe(map((name) => name ?? userId)),
+          );

          return new UserMedia(
            scope,
@@ -446,7 +439,7 @@ export class CallViewModel {
            this.mediaDevices,
            this.pretendToBeDisconnected$,
            displayName$,
-            mxcAvatarUrl$,
+            this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
            this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
            this.reactions$.pipe(map((v) => v[participantId] ?? undefined)),
          );
--- a/src/state/CallViewModel/localMember/LocalMembership.ts
+++ b/src/state/CallViewModel/localMember/LocalMembership.ts
@@ -43,14 +43,17 @@ import { type MuteStates } from "../../MuteStates";
 import { type ProcessorState } from "../../../livekit/TrackProcessorContext";
 import { type MediaDevices } from "../../MediaDevices";
 import { and$ } from "../../../utils/observable";
-import {
-  enterRTCSession,
-  type EnterRTCSessionOptions,
-} from "../../../rtcSessionHelpers";
 import { type ElementCallError } from "../../../utils/errors";
-import { ElementWidgetActions, type WidgetHelpers } from "../../../widget";
+import {
+  ElementWidgetActions,
+  widget,
+  type WidgetHelpers,
+} from "../../../widget";
 import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers";
 import { getUrlParams } from "../../../UrlParams.ts";
+import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
+import { MatrixRTCMode } from "../../../settings/settings.ts";
+import { Config } from "../../../config/Config.ts";

 export enum LivekitState {
  Uninitialized = "uninitialized",
@@ -535,3 +538,65 @@ export function observeSharingScreen$(p: Participant): Observable<boolean> {
    ParticipantEvent.LocalTrackUnpublished,
  ).pipe(map((p) => p.isScreenShareEnabled));
 }
+
+interface EnterRTCSessionOptions {
+  encryptMedia: boolean;
+  matrixRTCMode: MatrixRTCMode;
+}
+
+/**
+ * TODO! document this function properly
+ * @param rtcSession
+ * @param transport
+ * @param options
+ */
+async function enterRTCSession(
+  rtcSession: MatrixRTCSession,
+  transport: LivekitTransport,
+  { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
+): Promise<void> {
+  PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
+  PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
+
+  // This must be called before we start trying to join the call, as we need to
+  // have started tracking by the time calls start getting created.
+  // groupCallOTelMembership?.onJoinCall();
+
+  const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
+  const useDeviceSessionMemberEvents =
+    features?.feature_use_device_session_member_events;
+  const { sendNotificationType: notificationType, callIntent } = getUrlParams();
+  const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
+  // Multi-sfu does not need a preferred foci list. just the focus that is actually used.
+  rtcSession.joinRoomSession(
+    multiSFU ? [] : [transport],
+    multiSFU ? transport : undefined,
+    {
+      notificationType,
+      callIntent,
+      manageMediaKeys: encryptMedia,
+      ...(useDeviceSessionMemberEvents !== undefined && {
+        useLegacyMemberEvents: !useDeviceSessionMemberEvents,
+      }),
+      delayedLeaveEventRestartMs:
+        matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
+      delayedLeaveEventDelayMs:
+        matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
+      delayedLeaveEventRestartLocalTimeoutMs:
+        matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
+      networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
+      makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
+      membershipEventExpiryMs:
+        matrixRtcSessionConfig?.membership_event_expiry_ms,
+      useExperimentalToDeviceTransport: true,
+      unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
+    },
+  );
+  if (widget) {
+    try {
+      await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
+    } catch (e) {
+      logger.error("Failed to send join action", e);
+    }
+  }
+}
--- a/src/state/CallViewModel/localMember/Publisher.ts
+++ b/src/state/CallViewModel/localMember/Publisher.ts
@@ -74,6 +74,12 @@ export class Publisher {
    this.observeMediaDevices(scope, devices, controlledAudioDevices);

    this.workaroundRestartAudioInputTrackChrome(devices, scope);
+    this.scope.onEnd(() => {
+      this.logger?.info(
+        "[PublishConnection] Scope ended -> stop publishing all tracks",
+      );
+      void this.stopPublishing();
+    });
  }

  /**
--- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
+++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
@@ -13,19 +13,17 @@ import {
  type LivekitTransport,
  type CallMembership,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { combineLatest, filter, fromEvent, map, startWith } from "rxjs";
-// eslint-disable-next-line rxjs/no-internal
-import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
-import { RoomStateEvent, type Room as MatrixRoom } from "matrix-js-sdk";
-import { logger } from "matrix-js-sdk/lib/logger";
+import { combineLatest, filter, map } from "rxjs";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";

 import { type Behavior } from "../../Behavior";
 import { type IConnectionManager } from "./ConnectionManager";
 import { Epoch, type ObservableScope } from "../../ObservableScope";
-import { memberDisplaynames$ } from "./displayname";
 import { type Connection } from "./Connection";
 import { generateItemsWithEpoch } from "../../../utils/observable";

+const logger = rootLogger.getChild("MatrixLivekitMembers");
+
 /**
 * Represents a Matrix call member and their associated LiveKit participation.
 * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
@@ -39,8 +37,6 @@ export interface MatrixLivekitMember {
    LocalLivekitParticipant | RemoteLivekitParticipant | null
  >;
  connection$: Behavior<Connection | undefined>;
-  displayName$: Behavior<string>;
-  mxcAvatarUrl$: Behavior<string | undefined>;
 }

 interface Props {
@@ -49,16 +45,7 @@ interface Props {
    Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
  >;
  connectionManager: IConnectionManager;
-  // TODO this is too much information for that class,
-  // apparently needed to get a room member to later get the Avatar
-  // => Extract an AvatarService instead?
-  // Better with just `getMember`
-  matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter;
-  // roomMember$: Behavior<Pick<RoomMember, "userId" | "getMxcAvatarUrl">>;
 }
-// Alternative structure idea:
-// const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable<MatrixLivekitMember[]> => {
-
 /**
 * Combines MatrixRTC and Livekit worlds.
 *
@@ -73,22 +60,11 @@ export function createMatrixLivekitMembers$({
  scope,
  membershipsWithTransport$,
  connectionManager,
-  matrixRoom,
 }: Props): Behavior<Epoch<MatrixLivekitMember[]>> {
  /**
   * Stream of all the call members and their associated livekit data (if available).
   */

-  const displaynameMap$ = memberDisplaynames$(
-    scope,
-    matrixRoom,
-    scope.behavior(
-      membershipsWithTransport$.pipe(
-        map((ms) => ms.value.map((m) => m.membership)),
-      ),
-    ),
-  );
-
  return scope.behavior(
    combineLatest([
      membershipsWithTransport$,
@@ -130,29 +106,15 @@ export function createMatrixLivekitMembers$({
        },
        // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
        (scope, data$, participantId, userId) => {
-          const member = matrixRoom.getMember(userId);
+          logger.debug(
+            `Updating data$ for participantId: ${participantId}, userId: ${userId}`,
+          );
          // will only get called once per `participantId, userId` pair.
          // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
          return {
            participantId,
            userId,
            ...scope.splitBehavior(data$),
-            displayName$: scope.behavior(
-              displaynameMap$.pipe(
-                map((displayNames) => {
-                  const name = displayNames.get(userId) ?? "";
-                  if (name === "")
-                    logger.warn(`No display name for user ${userId}`);
-                  return name;
-                }),
-              ),
-            ),
-            mxcAvatarUrl$: scope.behavior(
-              fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
-                startWith(undefined),
-                map(() => member?.getMxcAvatarUrl()),
-              ),
-            ),
          };
        },
      ),
--- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
+++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
@@ -0,0 +1,148 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
+import { combineLatest, fromEvent, map } from "rxjs";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
+import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
+// eslint-disable-next-line rxjs/no-internal
+
+import { type ObservableScope } from "../../ObservableScope";
+import {
+  calculateDisplayName,
+  shouldDisambiguate,
+} from "../../../utils/displayname";
+import { type Behavior } from "../../Behavior";
+
+const logger = rootLogger.getChild("[MatrixMemberMetadata]");
+
+export type RoomMemberMap = Map<
+  string,
+  Pick<RoomMember, "userId" | "getMxcAvatarUrl" | "rawDisplayName">
+>;
+export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap {
+  return matrixRoom.getMembers().reduce((acc, member) => {
+    acc.set(member.userId, {
+      userId: member.userId,
+      getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member),
+      rawDisplayName: member.rawDisplayName,
+    });
+    return acc;
+  }, new Map());
+}
+
+export function createRoomMembers$(
+  scope: ObservableScope,
+  matrixRoom: MatrixRoom,
+): Behavior<RoomMemberMap> {
+  return scope.behavior(
+    fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
+      map(() => roomToMembersMap(matrixRoom)),
+    ),
+    roomToMembersMap(matrixRoom),
+  );
+}
+/**
+ * Displayname for each member of the call. This will disambiguate
+ * any displayname that clashes with another member. Only members
+ * joined to the call are considered here.
+ *
+ * @returns Map<userId, displayname> uses the Matrix user ID as the key.
+ */
+// don't do this work more times than we need to. This is achieved by converting to a behavior:
+export const memberDisplaynames$ = (
+  scope: ObservableScope,
+  memberships$: Behavior<Pick<CallMembership, "userId">[]>,
+  roomMembers$: Behavior<RoomMemberMap>,
+): Behavior<Map<string, string>> => {
+  // This map tracks userIds that at some point needed disambiguation.
+  // This is a memory leak bound to the number of participants.
+  // A call application will always increase the memory if there have been more members in a call.
+  // Its capped by room member participants.
+  const shouldDisambiguateTrackerMap = new Set<string>();
+  return scope.behavior(
+    combineLatest([
+      // Handle call membership changes
+      memberships$,
+      // Additionally handle display name changes (implicitly reacting to them)
+      roomMembers$,
+      // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
+    ]).pipe(
+      map(([memberships, roomMembers]) => {
+        const displaynameMap = new Map<string, string>();
+        // We only consider RTC members for disambiguation as they are the only visible members.
+        for (const rtcMember of memberships) {
+          const member = roomMembers.get(rtcMember.userId);
+          if (!member) {
+            logger.error(`Could not find member for user ${rtcMember.userId}`);
+            continue;
+          }
+          const disambiguateComputed = shouldDisambiguate(
+            member,
+            memberships,
+            roomMembers,
+          );
+
+          const disambiguate =
+            shouldDisambiguateTrackerMap.has(rtcMember.userId) ||
+            disambiguateComputed;
+          if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId);
+          displaynameMap.set(
+            rtcMember.userId,
+            calculateDisplayName(member, disambiguate),
+          );
+        }
+        return displaynameMap;
+      }),
+    ),
+  );
+};
+
+export const createMatrixMemberMetadata$ = (
+  scope: ObservableScope,
+  memberships$: Behavior<Pick<CallMembership, "userId">[]>,
+  roomMembers$: Behavior<RoomMemberMap>,
+): {
+  createDisplayNameBehavior$: (userId: string) => Behavior<string | undefined>;
+  createAvatarUrlBehavior$: (userId: string) => Behavior<string | undefined>;
+  displaynameMap$: Behavior<Map<string, string>>;
+  avatarMap$: Behavior<Map<string, string | undefined>>;
+} => {
+  const displaynameMap$ = memberDisplaynames$(
+    scope,
+    memberships$,
+    roomMembers$,
+  );
+  const avatarMap$ = scope.behavior(
+    roomMembers$.pipe(
+      map((roomMembers) =>
+        Array.from(roomMembers.keys()).reduce((acc, key) => {
+          acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl());
+          return acc;
+        }, new Map<string, string | undefined>()),
+      ),
+    ),
+  );
+  return {
+    createDisplayNameBehavior$: (userId: string) =>
+      scope.behavior(
+        displaynameMap$.pipe(
+          map((displaynameMap) => displaynameMap.get(userId)),
+        ),
+      ),
+    createAvatarUrlBehavior$: (userId: string) =>
+      scope.behavior(
+        roomMembers$.pipe(
+          map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),
+        ),
+      ),
+    // mostly for testing purposes
+    displaynameMap$,
+    avatarMap$,
+  };
+};
--- a/src/state/CallViewModel/remoteMembers/displayname.ts
+++ b/src/state/CallViewModel/remoteMembers/displayname.ts
@@ -1,76 +0,0 @@
-/*
-Copyright 2025 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
-import { combineLatest, fromEvent, map, startWith } from "rxjs";
-import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
-import { logger } from "matrix-js-sdk/lib/logger";
-import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
-// eslint-disable-next-line rxjs/no-internal
-import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
-
-import { type ObservableScope } from "../../ObservableScope";
-import {
-  calculateDisplayName,
-  shouldDisambiguate,
-} from "../../../utils/displayname";
-import { type Behavior } from "../../Behavior";
-
-export function createRoomMembers$(
-  scope: ObservableScope,
-  matrixRoom: MatrixRoom,
-): Behavior<Pick<RoomMember, "userId" | "getMxcAvatarUrl">[]> {
-  return scope.behavior(
-    fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
-      map(() => matrixRoom.getMembers()),
-    ),
-    [],
-  );
-}
-/**
- * Displayname for each member of the call. This will disambiguate
- * any displayname that clashes with another member. Only members
- * joined to the call are considered here.
- *
- * @returns Map<userId, displayname> uses the Matrix user ID as the key.
- */
-// don't do this work more times than we need to. This is achieved by converting to a behavior:
-export const memberDisplaynames$ = (
-  scope: ObservableScope,
-  matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter,
-  // roomMember$: Behavior<Pick<RoomMember, "userId" | "getMxcAvatarUrl">>;
-  memberships$: Behavior<CallMembership[]>,
-): Behavior<Map<string, string>> =>
-  scope.behavior(
-    combineLatest([
-      // Handle call membership changes
-      memberships$,
-      // Additionally handle display name changes (implicitly reacting to them)
-      fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)),
-      // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
-    ]).pipe(
-      map(([memberships, _displayNames]) => {
-        const displaynameMap = new Map<string, string>();
-        const room = matrixRoom;
-
-        // We only consider RTC members for disambiguation as they are the only visible members.
-        for (const rtcMember of memberships) {
-          const member = room.getMember(rtcMember.userId);
-          if (member === null) {
-            logger.error(`Could not find member for user ${rtcMember.userId}`);
-            continue;
-          }
-          const disambiguate = shouldDisambiguate(member, memberships, room);
-          displaynameMap.set(
-            rtcMember.userId,
-            calculateDisplayName(member, disambiguate),
-          );
-        }
-        return displaynameMap;
-      }),
-    ),
-  );