Add developer mode option to show RTC connection statistics (#2904)

* Add developer mode option to show RTC connection statistics

* Add note about localization

* Add titles to help explain what the numbers are

* Workaround horizontal scrolling

* Use modal to show detailed stats instead of alert

* Changed styling and fixed fps = 0 (#2916)

(React rendered 0 instead of <Text /> for fps && <Text>{fps}</text>)

---------

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

locales/en/app.json

@@ -73,6 +73,7 @@
     "duplicate_tiles_label": "Number of additional tile copies per participant",
     "hostname": "Hostname: {{hostname}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "show_connection_stats": "Show connection statistics",
     "show_non_member_tiles": "Show tiles for non-member media"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",

src/RTCConnectionStats.module.css

@@ -0,0 +1,20 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+.modal pre {
+  font-size: var(--font-size-micro);
+}
+
+.statsPill {
+  border-radius: var(--media-view-border-radius);
+  grid-area: none;
+  position: absolute;
+  top: 0;
+  left: 0;
+  flex-direction: column;
+  align-items: flex-start;
+}

src/RTCConnectionStats.tsx

@@ -0,0 +1,112 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { useState, type FC } from "react";
+import { Button, Text } from "@vector-im/compound-web";
+import {
+  MicOnSolidIcon,
+  VideoCallSolidIcon,
+} from "@vector-im/compound-design-tokens/assets/web/icons";
+import classNames from "classnames";
+
+import { Modal } from "./Modal";
+import styles from "./RTCConnectionStats.module.css";
+import mediaViewStyles from "../src/tile/MediaView.module.css";
+interface Props {
+  audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+}
+
+// This is only used in developer mode for debugging purposes, so we don't need full localization
+export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
+  const [showModal, setShowModal] = useState(false);
+  const [modalContents, setModalContents] = useState<
+    "video" | "audio" | "none"
+  >("none");
+
+  const showFullModal = (contents: "video" | "audio"): void => {
+    setShowModal(true);
+    setModalContents(contents);
+  };
+
+  const onDismissModal = (): void => {
+    setShowModal(false);
+    setModalContents("none");
+  };
+  return (
+    <div className={classNames(mediaViewStyles.nameTag, styles.statsPill)}>
+      <Modal
+        title="RTC Connection Stats"
+        open={showModal}
+        onDismiss={onDismissModal}
+      >
+        <div className={styles.modal}>
+          <pre>
+            {modalContents !== "none" &&
+              JSON.stringify(
+                modalContents === "video" ? video : audio,
+                null,
+                2,
+              )}
+          </pre>
+        </div>
+      </Modal>
+      {audio && (
+        <div>
+          <Button
+            onClick={() => showFullModal("audio")}
+            size="sm"
+            kind="tertiary"
+            Icon={MicOnSolidIcon}
+          >
+            {"jitter" in audio && typeof audio.jitter === "number" && (
+              <Text as="span" size="xs" title="jitter">
+                &nbsp;{(audio.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+          </Button>
+        </div>
+      )}
+      {video && (
+        <div>
+          <Button
+            onClick={() => showFullModal("video")}
+            size="sm"
+            kind="tertiary"
+            Icon={VideoCallSolidIcon}
+          >
+            {!!video?.framesPerSecond && (
+              <Text as="span" size="xs" title="frame rate">
+                &nbsp;{video.framesPerSecond.toFixed(0)}fps
+              </Text>
+            )}
+            {"jitter" in video && typeof video.jitter === "number" && (
+              <Text as="span" size="xs" title="jitter">
+                &nbsp;{(video.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+            {"frameHeight" in video &&
+              typeof video.frameHeight === "number" &&
+              "frameWidth" in video &&
+              typeof video.frameWidth === "number" && (
+                <Text as="span" size="xs" title="frame size">
+                  &nbsp;{video.frameWidth}x{video.frameHeight}
+                </Text>
+              )}
+            {"qualityLimitationReason" in video &&
+              typeof video.qualityLimitationReason === "string" &&
+              video.qualityLimitationReason !== "none" && (
+                <Text as="span" size="xs" title="quality limitation reason">
+                  &nbsp;{video.qualityLimitationReason}
+                </Text>
+              )}
+          </Button>
+        </div>
+      )}
+    </div>
+  );
+};

src/settings/DeveloperSettingsTab.tsx

@@ -14,6 +14,7 @@ import {
   duplicateTiles as duplicateTilesSetting,
   debugTileLayout as debugTileLayoutSetting,
   showNonMemberTiles as showNonMemberTilesSetting,
+  showConnectionStats as showConnectionStatsSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk/src/client";
 
@@ -31,6 +32,10 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
     showNonMemberTilesSetting,
   );
 
+  const [showConnectionStats, setShowConnectionStats] = useSetting(
+    showConnectionStatsSetting,
+  );
+
   return (
     <>
       <p>
@@ -103,6 +108,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="showConnectionStats"
+          type="checkbox"
+          label={t("developer_mode.show_connection_stats")}
+          checked={!!showConnectionStats}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setShowConnectionStats(event.target.checked);
+            },
+            [setShowConnectionStats],
+          )}
+        />
+      </FieldRow>
     </>
   );
 };

src/settings/settings.ts

@@ -78,6 +78,11 @@ export const showNonMemberTiles = new Setting<boolean>(
 );
 export const debugTileLayout = new Setting("debug-tile-layout", false);
 
+export const showConnectionStats = new Setting<boolean>(
+  "show-connection-stats",
+  false,
+);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,

src/state/MediaViewModel.ts

@@ -47,7 +47,7 @@ import { useEffect } from "react";
 
 import { ViewModel } from "./ViewModel";
 import { useReactiveState } from "../useReactiveState";
-import { alwaysShowSelf } from "../settings/settings";
+import { alwaysShowSelf, showConnectionStats } from "../settings/settings";
 import { accumulate } from "../utils/observable";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
@@ -97,6 +97,60 @@ export function observeTrackReference$(
   );
 }
 
+export function observeRtpStreamStats$(
+  participant: Participant,
+  source: Track.Source,
+  type: "inbound-rtp" | "outbound-rtp",
+): Observable<
+  RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+> {
+  return combineLatest([
+    observeTrackReference$(of(participant), source),
+    interval(1000).pipe(startWith(0)),
+  ]).pipe(
+    switchMap(async ([trackReference]) => {
+      const track = trackReference?.publication?.track;
+      if (
+        !track ||
+        !(track instanceof RemoteTrack || track instanceof LocalTrack)
+      ) {
+        return undefined;
+      }
+      const report = await track.getRTCStatsReport();
+      if (!report) {
+        return undefined;
+      }
+
+      for (const v of report.values()) {
+        if (v.type === type) {
+          return v;
+        }
+      }
+
+      return undefined;
+    }),
+    startWith(undefined),
+  );
+}
+
+export function observeInboundRtpStreamStats$(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCInboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe(
+    map((x) => x as RTCInboundRtpStreamStats | undefined),
+  );
+}
+
+export function observeOutboundRtpStreamStats$(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCOutboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe(
+    map((x) => x as RTCOutboundRtpStreamStats | undefined),
+  );
+}
+
 function observeRemoteTrackReceivingOkay$(
   participant: Participant,
   source: Track.Source,
@@ -111,33 +165,15 @@ function observeRemoteTrackReceivingOkay$(
     framesReceived: undefined,
   };
 
-  return combineLatest([
+  return observeInboundRtpStreamStats$(participant, source).pipe(
-    observeTrackReference$(of(participant), source),
+    map((stats) => {
-    interval(1000).pipe(startWith(0)),
+      if (!stats) return undefined;
-  ]).pipe(
+      const { framesDecoded, framesDropped, framesReceived } = stats;
-    switchMap(async ([trackReference]) => {
+      return {
-      const track = trackReference?.publication?.track;
+        framesDecoded,
-      if (!track || !(track instanceof RemoteTrack)) {
+        framesDropped,
-        return undefined;
+        framesReceived,
-      }
+      };
-      const report = await track.getRTCStatsReport();
-      if (!report) {
-        return undefined;
-      }
-
-      for (const v of report.values()) {
-        if (v.type === "inbound-rtp") {
-          const { framesDecoded, framesDropped, framesReceived } =
-            v as RTCInboundRtpStreamStats;
-          return {
-            framesDecoded,
-            framesDropped,
-            framesReceived,
-          };
-        }
-      }
-
-      return undefined;
     }),
     filter((newStats) => !!newStats),
     map((newStats): boolean | undefined => {
@@ -404,6 +440,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   public get local(): boolean {
     return this instanceof LocalUserMediaViewModel;
   }
+
+  public abstract get audioStreamStats$(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
+  public abstract get videoStreamStats$(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
 }
 
 /**
@@ -453,6 +496,26 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
       reaction$,
     );
   }
+
+  public audioStreamStats$ = combineLatest([
+    this.participant$,
+    showConnectionStats.value$,
+  ]).pipe(
+    switchMap(([p, showConnectionStats]) => {
+      if (!p || !showConnectionStats) return of(undefined);
+      return observeOutboundRtpStreamStats$(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats$ = combineLatest([
+    this.participant$,
+    showConnectionStats.value$,
+  ]).pipe(
+    switchMap(([p, showConnectionStats]) => {
+      if (!p || !showConnectionStats) return of(undefined);
+      return observeOutboundRtpStreamStats$(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**
@@ -542,6 +605,26 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public commitLocalVolume(): void {
     this.localVolumeCommit$.next();
   }
+
+  public audioStreamStats$ = combineLatest([
+    this.participant$,
+    showConnectionStats.value$,
+  ]).pipe(
+    switchMap(([p, showConnectionStats]) => {
+      if (!p || !showConnectionStats) return of(undefined);
+      return observeInboundRtpStreamStats$(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats$ = combineLatest([
+    this.participant$,
+    showConnectionStats.value$,
+  ]).pipe(
+    switchMap(([p, showConnectionStats]) => {
+      if (!p || !showConnectionStats) return of(undefined);
+      return observeInboundRtpStreamStats$(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**

src/tile/GridTile.tsx

@@ -86,6 +86,12 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     const video = useObservableEagerState(vm.video$);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning$);
     const encryptionStatus = useObservableEagerState(vm.encryptionStatus$);
+    const audioStreamStats = useObservableEagerState<
+      RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+    >(vm.audioStreamStats$);
+    const videoStreamStats = useObservableEagerState<
+      RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+    >(vm.videoStreamStats$);
     const audioEnabled = useObservableEagerState(vm.audioEnabled$);
     const videoEnabled = useObservableEagerState(vm.videoEnabled$);
     const speaking = useObservableEagerState(vm.speaking$);
@@ -174,6 +180,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         currentReaction={reaction ?? undefined}
         raisedHandOnClick={raisedHandOnClick}
         localParticipant={vm.local}
+        audioStreamStats={audioStreamStats}
+        videoStreamStats={videoStreamStats}
         {...props}
       />
     );

src/tile/MediaView.tsx

@@ -22,6 +22,7 @@ import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
 import { showHandRaisedTimer, useSetting } from "../settings/settings";
 import { type ReactionOption } from "../reactions";
 import { ReactionIndicator } from "../reactions/ReactionIndicator";
+import { RTCConnectionStats } from "../RTCConnectionStats";
 
 interface Props extends ComponentProps<typeof animated.div> {
   className?: string;
@@ -42,6 +43,8 @@ interface Props extends ComponentProps<typeof animated.div> {
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
   localParticipant: boolean;
+  audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
 }
 
 export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -65,6 +68,8 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       currentReaction,
       raisedHandOnClick,
       localParticipant,
+      audioStreamStats,
+      videoStreamStats,
       ...props
     },
     ref,
@@ -125,6 +130,12 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
               {t("video_tile.waiting_for_media")}
             </div>
           )}
+          {(audioStreamStats || videoStreamStats) && (
+            <RTCConnectionStats
+              audio={audioStreamStats}
+              video={videoStreamStats}
+            />
+          )}
           {/* TODO: Bring this back once encryption status is less broken */}
           {/*encryptionStatus !== EncryptionStatus.Okay && (
             <div className={styles.status}>