pull out all screen share related logic.

src/room/InCallView.tsx

@@ -58,7 +58,10 @@ import { type MuteStates } from "../state/MuteStates";
 import { type MatrixInfo } from "./VideoPreview";
 import { InviteButton } from "../button/InviteButton";
 import { LayoutToggle } from "./LayoutToggle";
-import { CallViewModel, type GridMode } from "../state/CallViewModel";
+import {
+  CallViewModel,
+  type GridMode,
+} from "../state/CallViewModel/CallViewModel.ts";
 import { Grid, type TileProps } from "../grid/Grid";
 import { useInitial } from "../useInitial";
 import { SpotlightTile } from "../tile/SpotlightTile";

src/state/CallViewModel/CallNotificationLifecycle.ts

@@ -0,0 +1,210 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  type CallMembership,
+  type MatrixRTCSession,
+  MatrixRTCSessionEvent,
+  type MatrixRTCSessionEventHandlerMap,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  combineLatest,
+  concat,
+  endWith,
+  filter,
+  fromEvent,
+  ignoreElements,
+  map,
+  merge,
+  NEVER,
+  type Observable,
+  of,
+  pairwise,
+  startWith,
+  switchMap,
+  takeUntil,
+  timer,
+} from "rxjs";
+import {
+  type EventTimelineSetHandlerMap,
+  EventType,
+  type Room as MatrixRoom,
+  RoomEvent,
+} from "matrix-js-sdk";
+
+import { type Behavior } from "../Behavior";
+import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
+export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
+export type CallPickupState =
+  | "unknown"
+  | "ringing"
+  | "timeout"
+  | "decline"
+  | "success"
+  | null;
+export type CallNotificationWrapper = Parameters<
+  MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
+>;
+export function createSentCallNotification$(
+  scope: ObservableScope,
+  matrixRTCSession: MatrixRTCSession,
+): Behavior<CallNotificationWrapper | null> {
+  const sentCallNotification$ = scope.behavior(
+    fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification),
+    null,
+  ) as Behavior<CallNotificationWrapper | null>;
+  return sentCallNotification$;
+}
+
+export function createReceivedDecline$(
+  matrixRoom: MatrixRoom,
+): Observable<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>> {
+  return (
+    fromEvent(matrixRoom, RoomEvent.Timeline) as Observable<
+      Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
+    >
+  ).pipe(filter(([event]) => event.getType() === EventType.RTCDecline));
+}
+
+interface Props {
+  scope: ObservableScope;
+  memberships$: Behavior<Epoch<CallMembership[]>>;
+  sentCallNotification$: Observable<CallNotificationWrapper | null>;
+  receivedDecline$: Observable<
+    Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
+  >;
+  options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean };
+  localUser: { deviceId: string; userId: string };
+}
+/**
+ * @returns {callPickupState$, autoLeave$}
+ * `callPickupState$` The current call pickup state of the call.
+ *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
+ *     Then we can conclude if we were the first one to join or not.
+ *     This may also be set if we are disconnected.
+ *  - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
+ *  - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
+ *     The call failed. If desired this can be used as a trigger to exit the call.
+ *  - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
+ *  - null: EC is configured to never show any waiting for answer state.
+ *
+ * `autoLeave$` An observable that emits (null) when the call should be automatically left.
+ *  - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
+ *  - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
+ *  - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
+ *
+ */
+export function createCallNotificationLifecycle$({
+  scope,
+  memberships$,
+  sentCallNotification$,
+  receivedDecline$,
+  options,
+  localUser,
+}: Props): {
+  callPickupState$: Behavior<CallPickupState>;
+  autoLeave$: Observable<AutoLeaveReason>;
+} {
+  // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
+  const allOthersLeft$ = memberships$.pipe(
+    pairwise(),
+    filter(
+      ([{ value: prev }, { value: current }]) =>
+        current.every((m) => m.userId === localUser.userId) &&
+        prev.some((m) => m.userId !== localUser.userId),
+    ),
+    map(() => {}),
+  );
+
+  /**
+   * Whether some Matrix user other than ourself is joined to the call.
+   */
+  const someoneElseJoined$ = memberships$.pipe(
+    mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
+  ) as Behavior<Epoch<boolean>>;
+
+  /**
+   * Whenever the RTC session tells us that it intends to ring the remote
+   * participant's devices, this emits an Observable tracking the current state of
+   * that ringing process.
+   */
+  // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
+  // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
+  // A behavior will emit the latest observable with the running timer to new subscribers.
+  // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
+  // `ring$` would not be a behavior.
+  const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
+    scope.behavior(
+      sentCallNotification$.pipe(
+        filter(
+          (newAndLegacyEvents) =>
+            // only care about new events (legacy do not have decline pattern)
+            newAndLegacyEvents?.[0].notification_type === "ring",
+        ),
+        map((e) => e as CallNotificationWrapper),
+        switchMap(([notificastionEvent]) => {
+          const lifetimeMs = notificationEvent?.lifetime ?? 0;
+          return concat(
+            lifetimeMs === 0
+              ? // If no lifetime, skip the ring state
+                of(null)
+              : // Ring until lifetime ms have passed
+                timer(lifetimeMs).pipe(
+                  ignoreElements(),
+                  startWith("ringing" as const),
+                ),
+            // The notification lifetime has timed out, meaning ringing has likely
+            // stopped on all receiving clients.
+            of("timeout" as const),
+            // This makes sure we will not drop into the `endWith("decline" as const)` state
+            NEVER,
+          ).pipe(
+            takeUntil(
+              receivedDecline$.pipe(
+                filter(
+                  ([event]) =>
+                    event.getRelation()?.rel_type === "m.reference" &&
+                    event.getRelation()?.event_id ===
+                      notificationEvent.event_id &&
+                    event.getSender() !== localUser.userId,
+                ),
+              ),
+            ),
+            endWith("decline" as const),
+          );
+        }),
+      ),
+      null,
+    );
+
+  const callPickupState$ = scope.behavior(
+    options.waitForCallPickup === true
+      ? combineLatest(
+          [someoneElseJoined$, remoteRingState$],
+          (someoneElseJoined, ring) => {
+            if (someoneElseJoined) {
+              return "success" as const;
+            }
+            // Show the ringing state of the most recent ringing attempt.
+            // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown.
+            return ring ?? ("unknown" as const);
+          },
+        )
+      : NEVER,
+    null,
+  );
+
+  const autoLeave$ = merge(
+    options.autoLeaveWhenOthersLeft === true
+      ? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
+      : NEVER,
+    callPickupState$.pipe(
+      filter((state) => state === "timeout" || state === "decline"),
+    ),
+  );
+  return { autoLeave$, callPickupState$ };
+}

src/state/CallViewModel/CallViewModel.ts

@@ -7,29 +7,20 @@ Please see LICENSE in the repository root for full details.
 
 import {
   type BaseKeyProvider,
-  ConnectionState,
+  type ConnectionState,
   type E2EEOptions,
   ExternalE2EEKeyProvider,
   type Room as LivekitRoom,
   type RoomOptions,
 } from "livekit-client";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
-import {
+import { type Room as MatrixRoom } from "matrix-js-sdk";
-  type EventTimelineSetHandlerMap,
-  EventType,
-  type Room as MatrixRoom,
-  RoomEvent,
-} from "matrix-js-sdk";
 import {
   combineLatest,
-  concat,
   distinctUntilChanged,
   EMPTY,
-  endWith,
   filter,
-  from,
   fromEvent,
-  ignoreElements,
   map,
   merge,
   NEVER,
@@ -46,18 +37,12 @@ import {
   switchMap,
   switchScan,
   take,
-  takeUntil,
-  takeWhile,
   tap,
   throttleTime,
   timer,
 } from "rxjs";
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
-import {
+import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
-  type MatrixRTCSession,
-  MatrixRTCSessionEvent,
-  type MatrixRTCSessionEventHandlerMap,
-} from "matrix-js-sdk/lib/matrixrtc";
 import { type IWidgetApiRequest } from "matrix-widget-api";
 
 import {
@@ -66,39 +51,39 @@ import {
   type RemoteUserMediaViewModel,
   ScreenShareViewModel,
   type UserMediaViewModel,
-} from "./MediaViewModel";
+} from "../MediaViewModel";
-import { accumulate, generateKeyed$, pauseWhen } from "../utils/observable";
+import { accumulate, generateKeyed$, pauseWhen } from "../../utils/observable";
 import {
   duplicateTiles,
   MatrixRTCMode,
   matrixRTCMode,
   playReactionsSound,
   showReactions,
-} from "../settings/settings";
+} from "../../settings/settings";
-import { isFirefox } from "../Platform";
+import { isFirefox } from "../../Platform";
-import { setPipEnabled$ } from "../controls";
+import { setPipEnabled$ } from "../../controls";
-import { TileStore } from "./TileStore";
+import { TileStore } from "../TileStore";
-import { gridLikeLayout } from "./GridLikeLayout";
+import { gridLikeLayout } from "../GridLikeLayout";
-import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
+import { spotlightExpandedLayout } from "../SpotlightExpandedLayout";
-import { oneOnOneLayout } from "./OneOnOneLayout";
+import { oneOnOneLayout } from "../OneOnOneLayout";
-import { pipLayout } from "./PipLayout";
+import { pipLayout } from "../PipLayout";
-import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
 import {
   type RaisedHandInfo,
   type ReactionInfo,
   type ReactionOption,
-} from "../reactions";
+} from "../../reactions";
-import { shallowEquals } from "../utils/array";
+import { shallowEquals } from "../../utils/array";
-import { type MediaDevices } from "./MediaDevices";
+import { type MediaDevices } from "../MediaDevices";
-import { type Behavior, constant } from "./Behavior";
+import { type Behavior, constant } from "../Behavior";
-import { E2eeType } from "../e2ee/e2eeType";
+import { E2eeType } from "../../e2ee/e2eeType";
-import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
+import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
-import { type MuteStates } from "./MuteStates";
+import { type MuteStates } from "../MuteStates";
-import { getUrlParams } from "../UrlParams";
+import { getUrlParams } from "../../UrlParams";
-import { type ProcessorState } from "../livekit/TrackProcessorContext";
+import { type ProcessorState } from "../../livekit/TrackProcessorContext";
-import { ElementWidgetActions, widget } from "../widget";
+import { ElementWidgetActions, widget } from "../../widget";
-import { sharingScreen$, UserMedia } from "./UserMedia.ts";
+import { UserMedia } from "../UserMedia.ts";
-import { ScreenShare } from "./ScreenShare.ts";
+import { ScreenShare } from "../ScreenShare.ts";
 import {
   type GridLayoutMedia,
   type Layout,
@@ -107,18 +92,23 @@ import {
   type SpotlightExpandedLayoutMedia,
   type SpotlightLandscapeLayoutMedia,
   type SpotlightPortraitLayoutMedia,
-} from "./layout-types.ts";
+} from "../layout-types.ts";
-import { type ElementCallError } from "../utils/errors.ts";
+import { type ElementCallError } from "../../utils/errors.ts";
-import { type ObservableScope } from "./ObservableScope.ts";
+import { type ObservableScope } from "../ObservableScope.ts";
 import { createLocalMembership$ } from "./localMember/LocalMembership.ts";
 import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
 import {
   createMemberships$,
   membershipsAndTransports$,
-} from "./SessionBehaviors.ts";
+} from "../SessionBehaviors.ts";
 import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
 import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
 import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts";
+import {
+  createCallNotificationLifecycle$,
+  createReceivedDecline$,
+  createSentCallNotification$,
+} from "./CallNotificationLifecycle.ts";
 
 const logger = rootLogger.getChild("[CallViewModel]");
 //TODO
@@ -274,6 +264,23 @@ export class CallViewModel {
     options: this.connectOptions$,
   });
 
+  // ------------------------------------------------------------------------
+  // CallNotificationLifecycle
+  private sentCallNotification$ = createSentCallNotification$(
+    this.scope,
+    this.matrixRTCSession,
+  );
+  private receivedDecline$ = createReceivedDecline$(this.matrixRoom);
+
+  private callLifecycle = createCallNotificationLifecycle$({
+    scope: this.scope,
+    memberships$: this.memberships$,
+    sentCallNotification$: this.sentCallNotification$,
+    receivedDecline$: this.receivedDecline$,
+    options: this.options,
+    localUser: { userId: this.userId, deviceId: this.deviceId },
+  });
+
   /**
    * If there is a configuration error with the call (e.g. misconfigured E2EE).
    * This is a fatal error that prevents the call from being created/joined.
@@ -315,7 +322,7 @@ export class CallViewModel {
 
   public readonly audioParticipants$ = this.scope.behavior(
     this.matrixLivekitMembers$.pipe(
-      map((members) => members.map((m) => m.participant)),
+      map((members) => members.value.map((m) => m.participant)),
     ),
   );
 
@@ -350,7 +357,7 @@ export class CallViewModel {
       // Generate a collection of MediaItems from the list of expected (whether
       // present or missing) LiveKit participants.
       combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]),
-      ([matrixLivekitMembers, duplicateTiles], createOrGet) => {
+      ([{ value: matrixLivekitMembers }, duplicateTiles], createOrGet) => {
         const items: MediaItem[] = [];
 
         for (const {
@@ -455,129 +462,11 @@ export class CallViewModel {
    */
   // TODO KEEP THIS!! and adapt it to what our membershipManger returns
   public readonly participantCount$ = this.scope.behavior(
-    this.memberships$.pipe(map((ms) => ms.length)),
+    this.memberships$.pipe(map((ms) => ms.value.length)),
   );
 
-  // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
-  private readonly allOthersLeft$ = this.memberships$.pipe(
-    pairwise(),
-    filter(
-      ([prev, current]) =>
-        current.every((m) => m.userId === this.userId) &&
-        prev.some((m) => m.userId !== this.userId),
-    ),
-    map(() => {}),
-  );
-
-  private readonly didSendCallNotification$ = fromEvent(
-    this.matrixRTCSession,
-    MatrixRTCSessionEvent.DidSendCallNotification,
-  ) as Observable<
-    Parameters<
-      MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
-    >
-  >;
-
-  /**
-   * Whenever the RTC session tells us that it intends to ring the remote
-   * participant's devices, this emits an Observable tracking the current state of
-   * that ringing process.
-   */
-  // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
-  // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
-  // A behavior will emit the latest observable with the running timer to new subscribers.
-  // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
-  // `ring$` would not be a behavior.
-  private readonly ring$: Behavior<"ringing" | "timeout" | "decline" | null> =
-    this.scope.behavior(
-      this.didSendCallNotification$.pipe(
-        filter(
-          ([notificationEvent]) =>
-            notificationEvent.notification_type === "ring",
-        ),
-        switchMap(([notificationEvent]) => {
-          const lifetimeMs = notificationEvent?.lifetime ?? 0;
-          return concat(
-            lifetimeMs === 0
-              ? // If no lifetime, skip the ring state
-                of(null)
-              : // Ring until lifetime ms have passed
-                timer(lifetimeMs).pipe(
-                  ignoreElements(),
-                  startWith("ringing" as const),
-                ),
-            // The notification lifetime has timed out, meaning ringing has likely
-            // stopped on all receiving clients.
-            of("timeout" as const),
-            // This makes sure we will not drop into the `endWith("decline" as const)` state
-            NEVER,
-          ).pipe(
-            takeUntil(
-              (
-                fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable<
-                  Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
-                >
-              ).pipe(
-                filter(
-                  ([event]) =>
-                    event.getType() === EventType.RTCDecline &&
-                    event.getRelation()?.rel_type === "m.reference" &&
-                    event.getRelation()?.event_id ===
-                      notificationEvent.event_id &&
-                    event.getSender() !== this.userId,
-                ),
-              ),
-            ),
-            endWith("decline" as const),
-          );
-        }),
-      ),
-      null,
-    );
-
-  /**
-   * Whether some Matrix user other than ourself is joined to the call.
-   */
-  private readonly someoneElseJoined$ = this.memberships$.pipe(
-    map((ms) => ms.some((m) => m.userId !== this.userId)),
-  ) as Behavior<boolean>;
-
-  /**
-   * The current call pickup state of the call.
-   *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
-   *     Then we can conclude if we were the first one to join or not.
-   *     This may also be set if we are disconnected.
-   *  - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
-   *  - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
-   *     The call failed. If desired this can be used as a trigger to exit the call.
-   *  - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
-   *  - null: EC is configured to never show any waiting for answer state.
-   */
-  public readonly callPickupState$: Behavior<
-    "unknown" | "ringing" | "timeout" | "decline" | "success" | null
-  > = this.options.waitForCallPickup
-    ? this.scope.behavior<
-        "unknown" | "ringing" | "timeout" | "decline" | "success"
-      >(
-        combineLatest(
-          [this.livekitConnectionState$, this.someoneElseJoined$, this.ring$],
-          (livekitConnectionState, someoneElseJoined, ring) => {
-            if (livekitConnectionState === ConnectionState.Disconnected) {
-              // Do not ring until we're connected.
-              return "unknown" as const;
-            } else if (someoneElseJoined) {
-              return "success" as const;
-            }
-            // Show the ringing state of the most recent ringing attempt.
-            // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown.
-            return ring ?? ("unknown" as const);
-          },
-        ),
-      )
-    : constant(null);
-
   public readonly leaveSoundEffect$ = combineLatest([
-    this.callPickupState$,
+    this.callLifecycle.callPickupState$,
     this.userMedia$,
   ]).pipe(
     // Until the call is successful, do not play a leave sound.
@@ -594,16 +483,6 @@ export class CallViewModel {
     throttleTime(THROTTLE_SOUND_EFFECT_MS),
   );
 
-  // Public for testing
-  public readonly autoLeave$ = merge(
-    this.options.autoLeaveWhenOthersLeft
-      ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
-      : NEVER,
-    this.callPickupState$.pipe(
-      filter((state) => state === "timeout" || state === "decline"),
-    ),
-  );
-
   private readonly userHangup$ = new Subject<void>();
   public hangup(): void {
     this.userHangup$.next();
@@ -626,7 +505,7 @@ export class CallViewModel {
   public readonly leave$: Observable<
     "user" | "timeout" | "decline" | "allOthersLeft"
   > = merge(
-    this.autoLeave$,
+    this.callLifecycle.autoLeave$,
     merge(this.userHangup$, this.widgetHangup$).pipe(
       map(() => "user" as const),
     ),
@@ -717,6 +596,7 @@ export class CallViewModel {
 
   private readonly pip$ = this.scope.behavior<UserMediaViewModel | null>(
     combineLatest([
+      // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits
       this.screenShares$,
       this.spotlightSpeaker$,
       this.mediaItems$,
@@ -1298,47 +1178,16 @@ export class CallViewModel {
   /**
    * Whether we are sharing our screen.
    */
-  // TODO move to LocalMembership
+  // reassigned here to make it publicly accessible
-  public readonly sharingScreen$ = this.scope.behavior(
+  public readonly sharingScreen$ = this.localMembership.sharingScreen$;
-    from(this.localConnection$).pipe(
-      switchMap((c) =>
-        c?.state === "ready"
-          ? sharingScreen$(c.value.livekitRoom.localParticipant)
-          : of(false),
-      ),
-    ),
-  );
 
   /**
    * Callback for toggling screen sharing. If null, screen sharing is not
    * available.
    */
-  // TODO move to LocalMembership
+  // reassigned here to make it publicly accessible
   public readonly toggleScreenSharing =
-    "getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
+    this.localMembership.toggleScreenSharing;
-    !this.urlParams.hideScreensharing
-      ? (): void =>
-          // Once a connection is ready...
-          void this.localConnection$
-            .pipe(
-              takeWhile((c) => c !== null && c.state !== "error"),
-              switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)),
-              take(1),
-              this.scope.bind(),
-            )
-            // ...toggle screen sharing.
-            .subscribe(
-              (c) =>
-                void c.livekitRoom.localParticipant
-                  .setScreenShareEnabled(!this.sharingScreen$.value, {
-                    audio: true,
-                    selfBrowserSurface: "include",
-                    surfaceSwitching: "include",
-                    systemAudio: "include",
-                  })
-                  .catch(logger.error),
-            )
-      : null;
 
   public constructor(
     private readonly scope: ObservableScope,

src/state/CallViewModel/localMember/LocalMembership.ts

@@ -18,58 +18,63 @@ import {
   combineLatest,
   fromEvent,
   map,
+  NEVER,
   type Observable,
   of,
   scan,
   startWith,
   switchMap,
+  take,
+  takeWhile,
   tap,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-import { type Behavior } from "../Behavior";
+import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts";
+import { type Behavior } from "../../Behavior";
 import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
-import { ObservableScope } from "../ObservableScope";
+import { ObservableScope } from "../../ObservableScope";
 import { Publisher } from "./Publisher";
-import { type MuteStates } from "../MuteStates";
+import { type MuteStates } from "../../MuteStates";
-import { type ProcessorState } from "../../livekit/TrackProcessorContext";
+import { type ProcessorState } from "../../../livekit/TrackProcessorContext";
-import { type MediaDevices } from "../MediaDevices";
+import { type MediaDevices } from "../../MediaDevices";
-import { and$ } from "../../utils/observable";
+import { and$ } from "../../../utils/observable";
 import {
   enterRTCSession,
   type EnterRTCSessionOptions,
-} from "../../rtcSessionHelpers";
+} from "../../../rtcSessionHelpers";
-import { type ElementCallError } from "../../utils/errors";
+import { type ElementCallError } from "../../../utils/errors";
-import { ElementWidgetActions, type WidgetHelpers } from "../../widget";
+import { ElementWidgetActions, type WidgetHelpers } from "../../../widget";
 import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers";
+import { getUrlParams } from "../../../UrlParams.ts";
 
-enum LivekitState {
+export enum LivekitState {
-  UNINITIALIZED = "uninitialized",
+  Uninitialized = "uninitialized",
-  CONNECTING = "connecting",
+  Connecting = "connecting",
-  CONNECTED = "connected",
+  Connected = "connected",
-  ERROR = "error",
+  Error = "error",
-  DISCONNECTED = "disconnected",
+  Disconnected = "disconnected",
-  DISCONNECTING = "disconnecting",
+  Disconnecting = "disconnecting",
 }
 type LocalMemberLivekitState =
-  | { state: LivekitState.ERROR; error: string }
+  | { state: LivekitState.Error; error: string }
-  | { state: LivekitState.CONNECTED }
+  | { state: LivekitState.Connected }
-  | { state: LivekitState.CONNECTING }
+  | { state: LivekitState.Connecting }
-  | { state: LivekitState.UNINITIALIZED }
+  | { state: LivekitState.Uninitialized }
-  | { state: LivekitState.DISCONNECTED }
+  | { state: LivekitState.Disconnected }
-  | { state: LivekitState.DISCONNECTING };
+  | { state: LivekitState.Disconnecting };
 
-enum MatrixState {
+export enum MatrixState {
-  CONNECTED = "connected",
+  Connected = "connected",
-  DISCONNECTED = "disconnected",
+  Disconnected = "disconnected",
-  CONNECTING = "connecting",
+  Connecting = "connecting",
 }
 type LocalMemberMatrixState =
-  | { state: MatrixState.CONNECTED }
+  | { state: MatrixState.Connected }
-  | { state: MatrixState.CONNECTING }
+  | { state: MatrixState.Connecting }
-  | { state: MatrixState.DISCONNECTED };
+  | { state: MatrixState.Disconnected };
 
-export interface LocalMemberState {
+export interface LocalMemberConnectionState {
   livekit$: BehaviorSubject<LocalMemberLivekitState>;
   matrix$: BehaviorSubject<LocalMemberMatrixState>;
 }
@@ -107,9 +112,10 @@ interface Props {
  * @param param0
  * @returns
  *  - publisher: The handle to create tracks and publish them to the room.
- *  - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation)
+ *  - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
  *  - transport$: the transport object the ownMembership$ ended up using.
- *
+ *  - connectionState: the current connection state. Including matrix server and livekit server connection.
+ *  - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
  */
 export const createLocalMembership$ = ({
   scope,
@@ -125,21 +131,31 @@ export const createLocalMembership$ = ({
   widget,
 }: Props): {
   // publisher: Publisher
-  requestConnect: () => LocalMemberState;
+  requestConnect: () => LocalMemberConnectionState;
   startTracks: () => Behavior<LocalTrack[]>;
   requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
-  state: LocalMemberState; // TODO this is probably superseeded by joinState$
+  connectionState: LocalMemberConnectionState;
+  sharingScreen$: Behavior<boolean | undefined>;
+  toggleScreenSharing: (() => void) | null;
+
+  // deprecated fields
+  /** @deprecated use state instead*/
   homeserverConnected$: Behavior<boolean>;
+  /** @deprecated use state instead*/
   connected$: Behavior<boolean>;
+  // this needs to be discussed
+  /** @deprecated use state instead*/
   reconnecting$: Behavior<boolean>;
+  // also needs to be disccues
+  /** @deprecated use state instead*/
   configError$: Behavior<ElementCallError | null>;
 } => {
   const state = {
     livekit$: new BehaviorSubject<LocalMemberLivekitState>({
-      state: LivekitState.UNINITIALIZED,
+      state: LivekitState.Uninitialized,
     }),
     matrix$: new BehaviorSubject<LocalMemberMatrixState>({
-      state: MatrixState.DISCONNECTED,
+      state: MatrixState.Disconnected,
     }),
   };
 
@@ -271,23 +287,23 @@ export const createLocalMembership$ = ({
     return tracks$;
   };
 
-  const requestConnect = (): LocalMemberState => {
+  const requestConnect = (): LocalMemberConnectionState => {
     if (state.livekit$.value === null) {
       startTracks();
-      state.livekit$.next({ state: LivekitState.CONNECTING });
+      state.livekit$.next({ state: LivekitState.Connecting });
       combineLatest([publisher$, tracks$], (publisher, tracks) => {
         publisher
           ?.startPublishing()
           .then(() => {
-            state.livekit$.next({ state: LivekitState.CONNECTED });
+            state.livekit$.next({ state: LivekitState.Connected });
           })
           .catch((error) => {
-            state.livekit$.next({ state: LivekitState.ERROR, error });
+            state.livekit$.next({ state: LivekitState.Error, error });
           });
       });
     }
-    if (state.matrix$.value.state !== MatrixState.DISCONNECTED) {
+    if (state.matrix$.value.state !== MatrixState.Disconnected) {
-      state.matrix$.next({ state: MatrixState.CONNECTING });
+      state.matrix$.next({ state: MatrixState.Connecting });
       localTransport$.pipe(
         tap((transport) => {
           if (transport !== undefined) {
@@ -306,17 +322,17 @@ export const createLocalMembership$ = ({
   };
 
   const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
-    if (state.livekit$.value.state !== LivekitState.CONNECTED) return null;
+    if (state.livekit$.value.state !== LivekitState.Connected) return null;
-    state.livekit$.next({ state: LivekitState.DISCONNECTING });
+    state.livekit$.next({ state: LivekitState.Disconnecting });
     combineLatest([publisher$, tracks$], (publisher, tracks) => {
       publisher
         ?.stopPublishing()
         .then(() => {
           tracks.forEach((track) => track.stop());
-          state.livekit$.next({ state: LivekitState.DISCONNECTED });
+          state.livekit$.next({ state: LivekitState.Disconnected });
         })
         .catch((error) => {
-          state.livekit$.next({ state: LivekitState.ERROR, error });
+          state.livekit$.next({ state: LivekitState.Error, error });
         });
     });
 
@@ -410,14 +426,83 @@ export const createLocalMembership$ = ({
     }
   });
 
+  /**
+   * Returns undefined if scrennSharing is not yet ready.
+   */
+  const sharingScreen$ = scope.behavior(
+    connection$.pipe(
+      switchMap((c) => {
+        if (!c) return of(undefined);
+        if (c.state$.value.state === "ConnectedToLkRoom")
+          return observeSharingScreen$(c.livekitRoom.localParticipant);
+        return of(false);
+      }),
+    ),
+  );
+
+  const toggleScreenSharing =
+    "getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
+    !getUrlParams().hideScreensharing
+      ? (): void =>
+          // If a connection is ready...
+          void connection$
+            .pipe(
+              // I dont see why we need this. isnt the check later on superseeding it?
+              takeWhile(
+                (c) =>
+                  c !== undefined && c.state$.value.state !== "FailedToStart",
+              ),
+              switchMap((c) =>
+                c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER,
+              ),
+              take(1),
+              scope.bind(),
+            )
+            // ...toggle screen sharing.
+            .subscribe(
+              (c) =>
+                void c.livekitRoom.localParticipant
+                  .setScreenShareEnabled(!sharingScreen$.value, {
+                    audio: true,
+                    selfBrowserSurface: "include",
+                    surfaceSwitching: "include",
+                    systemAudio: "include",
+                  })
+                  .catch(logger.error),
+            )
+      : null;
+
+  // we do not need all the auto waiting since we can just check via sharingScreen$.value !== undefined
+  let alternativeScreenshareToggle: (() => void) | null = null;
+  if (
+    "getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
+    !getUrlParams().hideScreensharing
+  ) {
+    alternativeScreenshareToggle = (): void =>
+      void connection$.value?.livekitRoom.localParticipant
+        .setScreenShareEnabled(!sharingScreen$.value, {
+          audio: true,
+          selfBrowserSurface: "include",
+          surfaceSwitching: "include",
+          systemAudio: "include",
+        })
+        .catch(logger.error);
+  }
+  logger.log(
+    "alternativeScreenshareToggle so that it is used",
+    alternativeScreenshareToggle,
+  );
+
   return {
     startTracks,
     requestConnect,
     requestDisconnect,
-    state,
+    connectionState: state,
     homeserverConnected$,
     connected$,
     reconnecting$,
     configError$,
+    sharingScreen$,
+    toggleScreenSharing,
   };
 };

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -18,15 +18,15 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
 
-import { type Behavior } from "../Behavior.ts";
+import { type Behavior } from "../../Behavior.ts";
 import {
   type Epoch,
   mapEpoch,
   type ObservableScope,
-} from "../ObservableScope.ts";
+} from "../../ObservableScope.ts";
-import { Config } from "../../config/Config.ts";
+import { Config } from "../../../config/Config.ts";
-import { MatrixRTCTransportMissingError } from "../../utils/errors.ts";
+import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
-import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts";
+import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts";
 
 /*
  * - get well known

src/state/CallViewModel/localMember/Publisher.ts

@@ -22,17 +22,17 @@ import {
 } from "rxjs";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import type { Behavior } from "../Behavior.ts";
+import type { Behavior } from "../../Behavior.ts";
-import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts";
+import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
-import type { MuteStates } from "../MuteStates.ts";
+import type { MuteStates } from "../../MuteStates.ts";
 import {
   type ProcessorState,
   trackProcessorSync,
-} from "../../livekit/TrackProcessorContext.tsx";
+} from "../../../livekit/TrackProcessorContext.tsx";
-import { getUrlParams } from "../../UrlParams.ts";
+import { getUrlParams } from "../../../UrlParams.ts";
-import { observeTrackReference$ } from "../MediaViewModel.ts";
+import { observeTrackReference$ } from "../../MediaViewModel.ts";
-import { type Connection } from "../remoteMembers/Connection.ts";
+import { type Connection } from "../CallViewModel/remoteMembers/Connection.ts";
-import { type ObservableScope } from "../ObservableScope.ts";
+import { type ObservableScope } from "../../ObservableScope.ts";
 
 /**
  * A wrapper for a Connection object.

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -36,12 +36,12 @@ import {
   type ConnectionState,
   type PublishingParticipant,
 } from "./Connection.ts";
-import { ObservableScope } from "../ObservableScope.ts";
+import { ObservableScope } from "../../ObservableScope.ts";
-import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
+import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
-import { FailToGetOpenIdToken } from "../../utils/errors.ts";
+import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
-import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts";
+import { mockMediaDevices, mockMuteStates } from "../../../utils/test.ts";
-import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
+import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
-import { type MuteStates } from "../MuteStates.ts";
+import { type MuteStates } from "../../MuteStates.ts";
 
 let testScope: ObservableScope;
 

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -25,13 +25,13 @@ import {
   getSFUConfigWithOpenID,
   type OpenIDClientParts,
   type SFUConfig,
-} from "../../livekit/openIDSFU.ts";
+} from "../../../livekit/openIDSFU.ts";
-import { type Behavior } from "../Behavior.ts";
+import { type Behavior } from "../../Behavior.ts";
-import { type ObservableScope } from "../ObservableScope.ts";
+import { type ObservableScope } from "../../ObservableScope.ts";
 import {
   InsufficientCapacityError,
   SFURoomCreationRestrictedError,
-} from "../../utils/errors.ts";
+} from "../../../utils/errors.ts";
 
 export type PublishingParticipant = LocalParticipant | RemoteParticipant;
 

src/state/CallViewModel/remoteMembers/ConnectionFactory.ts

@@ -13,13 +13,13 @@ import {
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import { type ObservableScope } from "../ObservableScope.ts";
+import { type ObservableScope } from "../../ObservableScope.ts";
 import { Connection } from "./Connection.ts";
-import type { OpenIDClientParts } from "../../livekit/openIDSFU.ts";
+import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
-import type { MediaDevices } from "../MediaDevices.ts";
+import type { MediaDevices } from "../../MediaDevices.ts";
-import type { Behavior } from "../Behavior.ts";
+import type { Behavior } from "../../Behavior.ts";
-import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
+import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
-import { defaultLiveKitOptions } from "../../livekit/options.ts";
+import { defaultLiveKitOptions } from "../../../livekit/options.ts";
 
 export interface ConnectionFactory {
   createConnection(

src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts

@@ -10,14 +10,14 @@ import { BehaviorSubject } from "rxjs";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { type Participant as LivekitParticipant } from "livekit-client";
 
-import { ObservableScope } from "../ObservableScope.ts";
+import { ObservableScope } from "../../ObservableScope.ts";
 import {
   type IConnectionManager,
   createConnectionManager$,
 } from "./ConnectionManager.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 import { type Connection } from "./Connection.ts";
-import { flushPromises, withTestScheduler } from "../../utils/test.ts";
+import { flushPromises, withTestScheduler } from "../../../utils/test.ts";
 import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 
 // Some test constants

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -17,10 +17,10 @@ import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs";
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
 
-import { type Behavior } from "../Behavior";
+import { type Behavior } from "../../Behavior.ts";
-import { type Connection } from "./Connection";
+import { type Connection } from "./Connection.ts";
-import { Epoch, type ObservableScope } from "../ObservableScope";
+import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
-import { generateKeyed$ } from "../../utils/observable";
+import { generateKeyed$ } from "../../../utils/observable.ts";
 import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -19,14 +19,14 @@ import {
   createMatrixLivekitMembers$,
   areLivekitTransportsEqual,
 } from "./MatrixLivekitMembers.ts";
-import { ObservableScope } from "../ObservableScope.ts";
+import { ObservableScope } from "../../ObservableScope.ts";
 import { ConnectionManagerData } from "./ConnectionManager.ts";
 import {
   mockCallMembership,
   mockRemoteParticipant,
   type OurRunHelpers,
   withTestScheduler,
-} from "../../utils/test.ts";
+} from "../../../utils/test.ts";
 import { type Connection } from "./Connection.ts";
 
 let testScope: ObservableScope;

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -19,9 +19,9 @@ import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
 import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-import { type Behavior } from "../Behavior";
+import { type Behavior } from "../../Behavior";
 import { type IConnectionManager } from "./ConnectionManager";
-import { Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
+import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope";
 import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
 import { type Connection } from "./Connection";
 

src/state/CallViewModel/remoteMembers/displayname.test.ts

@@ -14,9 +14,9 @@ import {
 } from "matrix-js-sdk";
 import EventEmitter from "events";
 
-import { ObservableScope } from "../ObservableScope.ts";
+import { ObservableScope } from "../../ObservableScope.ts";
 import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
-import { mockCallMembership, withTestScheduler } from "../../utils/test.ts";
+import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts";
 import { memberDisplaynames$ } from "./displayname.ts";
 
 let testScope: ObservableScope;

src/state/CallViewModel/remoteMembers/displayname.ts

@@ -19,12 +19,12 @@ import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
 // eslint-disable-next-line rxjs/no-internal
 import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
 
-import { Epoch, type ObservableScope } from "../ObservableScope";
+import { Epoch, type ObservableScope } from "../../ObservableScope";
 import {
   calculateDisplayName,
   shouldDisambiguate,
-} from "../../utils/displayname";
+} from "../../../utils/displayname";
-import { type Behavior } from "../Behavior";
+import { type Behavior } from "../../Behavior";
 
 /**
  * Displayname for each member of the call. This will disambiguate

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -14,22 +14,26 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-import { type Epoch, ObservableScope, trackEpoch } from "../ObservableScope.ts";
+import {
+  type Epoch,
+  ObservableScope,
+  trackEpoch,
+} from "../../ObservableScope.ts";
 import { ECConnectionFactory } from "./ConnectionFactory.ts";
-import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
+import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
 import {
   mockCallMembership,
   mockMediaDevices,
   withTestScheduler,
-} from "../../utils/test";
+} from "../../../utils/test.ts";
-import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
+import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
 import {
   areLivekitTransportsEqual,
   createMatrixLivekitMembers$,
   type MatrixLivekitMember,
 } from "./MatrixLivekitMembers.ts";
 import { createConnectionManager$ } from "./ConnectionManager.ts";
-import { membershipsAndTransports$ } from "../SessionBehaviors.ts";
+import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
 
 // Test the integration of ConnectionManager and MatrixLivekitMerger