Simplify AudioRenderer and add more tests

src/livekit/MatrixAudioRenderer.test.tsx

@@ -14,8 +14,8 @@ import {
 import {
   type Participant,
   type RemoteAudioTrack,
-  type RemoteParticipant,
   type Room,
+  Track,
 } from "livekit-client";
 import { type ReactNode } from "react";
 import { useTracks } from "@livekit/components-react";
@@ -23,7 +23,11 @@ import { useTracks } from "@livekit/components-react";
 import { testAudioContext } from "../useAudioContext.test";
 import * as MediaDevicesContext from "../MediaDevicesContext";
 import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
-import { mockMediaDevices, mockTrack } from "../utils/test";
+import {
+  mockMediaDevices,
+  mockRemoteParticipant,
+  mockTrack,
+} from "../utils/test";
 
 export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
 
@@ -61,17 +65,20 @@ let tracks: TrackReference[] = [];
  *
  * @param rtcMembers - Array of active rtc members with userId and deviceId.
  * @param livekitParticipantIdentities - Array of livekit participant (that are publishing).
+ * @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created.
  * */
 
 function renderTestComponent(
   rtcMembers: { userId: string; deviceId: string }[],
   livekitParticipantIdentities: string[],
+  explicitTracks?: {
+    participantId: string;
+    kind: Track.Kind;
+    source: Track.Source;
+  }[],
 ): RenderResult {
-  const liveKitParticipants = livekitParticipantIdentities.map(
+  const liveKitParticipants = livekitParticipantIdentities.map((identity) =>
-    (identity) =>
+    mockRemoteParticipant({ identity }),
-      ({
-        identity,
-      }) as unknown as RemoteParticipant,
   );
   const participants = rtcMembers.flatMap(({ userId, deviceId }) => {
     const p = liveKitParticipants.find(
@@ -85,13 +92,22 @@ function renderTestComponent(
     ),
   } as unknown as Room;
 
-  tracks = participants.map((p) => mockTrack(p));
+  if (explicitTracks?.length ?? 0 > 0) {
+    tracks = explicitTracks!.map(({ participantId, source, kind }) => {
+      const participant =
+        liveKitParticipants.find((p) => p.identity === participantId) ??
+        mockRemoteParticipant({ identity: participantId });
+      return mockTrack(participant, kind, source);
+    });
+  } else {
+    tracks = participants.map((p) => mockTrack(p));
+  }
 
   vi.mocked(useTracks).mockReturnValue(tracks);
   return render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
       <LivekitRoomAudioRenderer
-        participants={participants}
+        validIdentities={participants.map((p) => p.identity)}
         livekitRoom={livekitRoom}
         url={""}
       />
@@ -118,11 +134,18 @@ it("should not render without member", () => {
 });
 
 const TEST_CASES: {
+  name: string;
   rtcUsers: { userId: string; deviceId: string }[];
   livekitParticipantIdentities: string[];
+  explicitTracks?: {
+    participantId: string;
+    kind: Track.Kind;
+    source: Track.Source;
+  }[];
   expectedAudioTracks: number;
 }[] = [
   {
+    name: "single user single device",
     rtcUsers: [
       { userId: "@alice", deviceId: "DEV0" },
       { userId: "@alice", deviceId: "DEV1" },
@@ -133,6 +156,7 @@ const TEST_CASES: {
   },
   // Charlie is a rtc member but not in livekit
   {
+    name: "Charlie is rtc member but not in livekit",
     rtcUsers: [
       { userId: "@alice", deviceId: "DEV0" },
       { userId: "@bob", deviceId: "DEV0" },
@@ -143,6 +167,7 @@ const TEST_CASES: {
   },
   // Charlie is in livekit but not rtc member
   {
+    name: "Charlie is in livekit but not rtc member",
     rtcUsers: [
       { userId: "@alice", deviceId: "DEV0" },
       { userId: "@bob", deviceId: "DEV0" },
@@ -150,14 +175,77 @@ const TEST_CASES: {
     livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"],
     expectedAudioTracks: 2,
   },
+  {
+    name: "no audio track, only video track",
+    rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
+    livekitParticipantIdentities: ["@alice:DEV0"],
+    explicitTracks: [
+      {
+        participantId: "@alice:DEV0",
+        kind: Track.Kind.Video,
+        source: Track.Source.Camera,
+      },
+    ],
+    expectedAudioTracks: 0,
+  },
+  {
+    name: "Audio track from unknown source",
+    rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
+    livekitParticipantIdentities: ["@alice:DEV0"],
+    explicitTracks: [
+      {
+        participantId: "@alice:DEV0",
+        kind: Track.Kind.Audio,
+        source: Track.Source.Unknown,
+      },
+    ],
+    expectedAudioTracks: 1,
+  },
+  {
+    name: "Audio track from other device",
+    rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
+    livekitParticipantIdentities: ["@alice:DEV0"],
+    explicitTracks: [
+      {
+        participantId: "@alice:DEV1",
+        kind: Track.Kind.Audio,
+        source: Track.Source.Microphone,
+      },
+    ],
+    expectedAudioTracks: 0,
+  },
+  {
+    name: "two audio tracks, microphone and screenshare",
+    rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
+    livekitParticipantIdentities: ["@alice:DEV0"],
+    explicitTracks: [
+      {
+        participantId: "@alice:DEV0",
+        kind: Track.Kind.Audio,
+        source: Track.Source.Microphone,
+      },
+      {
+        participantId: "@alice:DEV0",
+        kind: Track.Kind.Audio,
+        source: Track.Source.ScreenShareAudio,
+      },
+    ],
+    expectedAudioTracks: 2,
+  },
 ];
 
 it.each(TEST_CASES)(
-  `should render sound test cases %s`,
+  `should render sound test cases $name`,
-  ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }) => {
+  ({
+    rtcUsers,
+    livekitParticipantIdentities,
+    explicitTracks,
+    expectedAudioTracks,
+  }) => {
     const { queryAllByTestId } = renderTestComponent(
       rtcUsers,
       livekitParticipantIdentities,
+      explicitTracks,
     );
     expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks);
   },

src/livekit/MatrixAudioRenderer.tsx

@@ -6,23 +6,20 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { getTrackReferenceId } from "@livekit/components-core";
-import {
+import { type Room as LivekitRoom } from "livekit-client";
-  type RemoteParticipant,
-  type Room as LivekitRoom,
-} from "livekit-client";
 import { type RemoteAudioTrack, Track } from "livekit-client";
-import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
+import { useEffect, useMemo, useState, type ReactNode } from "react";
 import {
   useTracks,
   AudioTrack,
   type AudioTrackProps,
 } from "@livekit/components-react";
 import { logger } from "matrix-js-sdk/lib/logger";
+import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc";
 
 import { useEarpieceAudioConfig } from "../MediaDevicesContext";
 import { useReactiveState } from "../useReactiveState";
 import * as controls from "../controls";
-import {} from "@livekit/components-core";
 
 export interface MatrixAudioRendererProps {
   /**
@@ -31,11 +28,11 @@ export interface MatrixAudioRendererProps {
   url: string;
   livekitRoom: LivekitRoom;
   /**
-   * The list of participants to render audio for.
+   * The list of participant identities to render audio for.
    * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
-   * that are not expected to be in the rtc session.
+   * that are not expected to be in the rtc session (local user is excluded).
    */
-  participants: RemoteParticipant[];
+  validIdentities: ParticipantId[];
   /**
    * If set to `true`, mutes all audio tracks rendered by the component.
    * @remarks
@@ -44,6 +41,7 @@ export interface MatrixAudioRendererProps {
   muted?: boolean;
 }
 
+const prefixedLogger = logger.getChild("[MatrixAudioRenderer]");
 /**
  * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
  *
@@ -60,35 +58,9 @@ export interface MatrixAudioRendererProps {
 export function LivekitRoomAudioRenderer({
   url,
   livekitRoom,
-  participants,
+  validIdentities,
   muted,
 }: MatrixAudioRendererProps): ReactNode {
-  // This is the list of valid identities that are allowed to play audio.
-  // It is derived from the list of matrix rtc members.
-  const validIdentities = useMemo(
-    () => new Set(participants.map((p) => p.identity)),
-    [participants],
-  );
-
-  const loggedInvalidIdentities = useRef(new Set<string>());
-
-  /**
-   * Log an invalid livekit track identity.
-   * A invalid identity is one that does not match any of the matrix rtc members.
-   *
-   * @param identity The identity of the track that is invalid
-   * @param validIdentities The list of valid identities
-   */
-  const logInvalid = (identity: string): void => {
-    if (loggedInvalidIdentities.current.has(identity)) return;
-    logger.warn(
-      `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
-      `current members: ${participants.map((p) => p.identity)}`,
-      `track will not get rendered`,
-    );
-    loggedInvalidIdentities.current.add(identity);
-  };
-
   const tracks = useTracks(
     [
       Track.Source.Microphone,
@@ -100,28 +72,23 @@ export function LivekitRoomAudioRenderer({
       onlySubscribed: true,
       room: livekitRoom,
     },
-  ).filter((ref) => {
+  )
-    const isValid = validIdentities.has(ref.participant.identity);
+    // Only keep audio tracks
-    if (!isValid && !ref.participant.isLocal)
+    .filter((ref) => ref.publication.kind === Track.Kind.Audio)
-      logInvalid(ref.participant.identity);
+    // Only keep tracks from participants that are in the validIdentities list
-    return (
+    .filter((ref) => {
-      !ref.participant.isLocal &&
+      const isValid = validIdentities.includes(ref.participant.identity);
-      ref.publication.kind === Track.Kind.Audio &&
+      if (!isValid) {
-      isValid
+        // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call.
-    );
+        prefixedLogger.warn(
-  });
+          `Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`,
-
+          `current members: ${validIdentities.join()}`,
-  useEffect(() => {
+          `track will not get rendered`,
-    if (
+        );
-      loggedInvalidIdentities.current.size &&
+        return false;
-      tracks.every((t) => validIdentities.has(t.participant.identity))
+      }
-    ) {
+      return true;
-      logger.debug(
+    });
-        `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
-      );
-      loggedInvalidIdentities.current.clear();
-    }
-  }, [tracks, validIdentities, url]);
 
   // This component is also (in addition to the "only play audio for connected members" logic above)
   // responsible for mimicking earpiece audio on iPhones.

src/room/InCallView.tsx

@@ -865,7 +865,7 @@ export const InCallView: FC<InCallViewProps> = ({
           key={url}
           url={url}
           livekitRoom={livekitRoom}
-          participants={participants}
+          validIdentities={participants.map((p) => p.identity)}
           muted={muteAllAudio}
         />
       ))}

src/utils/test.ts

@@ -393,13 +393,17 @@ export class MockRTCSession extends TypedEventEmitter<
   }
 }
 
-export const mockTrack = (participant: Participant): TrackReference =>
+export const mockTrack = (
+  participant: Participant,
+  kind?: Track.Kind,
+  source?: Track.Source,
+): TrackReference =>
   ({
     participant,
     publication: {
-      kind: Track.Kind.Audio,
+      kind: kind ?? Track.Kind.Audio,
-      source: "mic",
+      source: source ?? Track.Source.Microphone,
-      trackSid: "123",
+      trackSid: `123##${participant.identity}`,
       track: {
         attach: vi.fn(),
         detach: vi.fn(),