Add sound effect for call joined / left (#2794)

* Add renderer for call joined / left

* lint

* Add new sounds

* Updates sounds in renderer

* lint

* move import

* pad sounds with silence

* lint

* tidy

* Drop autoplay since we now subscribe correctly.

* Comitting test files I am going to be going to lunch so will tidy up in a little while.

* finish up tests

* Add support for multiple channels per sound.

* lint

src/room/CallEventAudioRenderer.test.tsx

@@ -0,0 +1,225 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { render } from "@testing-library/react";
+import { beforeEach, expect, test } from "vitest";
+import { MatrixClient } from "matrix-js-sdk/src/client";
+import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
+import { of } from "rxjs";
+import { afterEach } from "node:test";
+import { act } from "react";
+
+import { soundEffectVolumeSetting } from "../settings/settings";
+import {
+  EmittableMockLivekitRoom,
+  mockLivekitRoom,
+  mockLocalParticipant,
+  mockMatrixRoom,
+  mockMatrixRoomMember,
+  mockMediaPlay,
+  mockRemoteParticipant,
+} from "../utils/test";
+import { E2eeType } from "../e2ee/e2eeType";
+import { CallViewModel } from "../state/CallViewModel";
+import {
+  CallEventAudioRenderer,
+  MAX_PARTICIPANT_COUNT_FOR_SOUND,
+} from "./CallEventAudioRenderer";
+
+const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
+const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
+const aliceId = `${alice.userId}:AAAA`;
+const bobId = `${bob.userId}:BBBB`;
+const localParticipant = mockLocalParticipant({ identity: "" });
+const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
+const bobParticipant = mockRemoteParticipant({ identity: bobId });
+
+const originalPlayFn = window.HTMLMediaElement.prototype.play;
+
+const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
+const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
+
+beforeEach(() => {
+  soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
+});
+
+afterEach(() => {
+  window.HTMLMediaElement.prototype.play = originalPlayFn;
+});
+
+test("plays a sound when entering a call", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const remoteParticipants = of([aliceParticipant]);
+  const liveKitRoom = mockLivekitRoom(
+    { localParticipant },
+    { remoteParticipants },
+  );
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+
+  render(<CallEventAudioRenderer vm={vm} />);
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+  ]);
+});
+
+test("plays no sound when muted", () => {
+  soundEffectVolumeSetting.setValue(0);
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const remoteParticipants = of([aliceParticipant, bobParticipant]);
+  const liveKitRoom = mockLivekitRoom(
+    { localParticipant },
+    { remoteParticipants },
+  );
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+
+  render(<CallEventAudioRenderer vm={vm} />);
+  // Play a sound when joining a call.
+  expect(audioIsPlaying).toHaveLength(0);
+});
+
+test("plays a sound when a user joins", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+
+  act(() => {
+    liveKitRoom.addParticipant(bobParticipant);
+  });
+  // Play a sound when joining a call.
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+    // Bob leaves
+    enterSound,
+  ]);
+});
+
+test("plays a sound when a user leaves", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+
+  act(() => {
+    liveKitRoom.removeParticipant(aliceParticipant);
+  });
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+    // Alice leaves
+    leaveSound,
+  ]);
+});
+
+test("plays no sound when the participant list", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map<string, RemoteParticipant>([
+    [aliceParticipant.identity, aliceParticipant],
+    ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
+      [string, RemoteParticipant]
+    >((_, index) => {
+      const p = mockRemoteParticipant({ identity: `user${index}` });
+      return [p.identity, p];
+    }),
+  ]);
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+  expect(audioIsPlaying).toEqual([]);
+  // When the count drops
+  act(() => {
+    liveKitRoom.removeParticipant(aliceParticipant);
+  });
+  expect(audioIsPlaying).toEqual([leaveSound]);
+});

src/room/CallEventAudioRenderer.tsx

@@ -0,0 +1,117 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { ReactNode, useEffect, useRef } from "react";
+import { filter } from "rxjs";
+
+import {
+  soundEffectVolumeSetting as effectSoundVolumeSetting,
+  useSetting,
+} from "../settings/settings";
+import { CallViewModel } from "../state/CallViewModel";
+import enterCallSoundMp3 from "../sound/join_call.mp3";
+import enterCallSoundOgg from "../sound/join_call.ogg";
+import leftCallSoundMp3 from "../sound/left_call.mp3";
+import leftCallSoundOgg from "../sound/left_call.ogg";
+
+// Do not play any sounds if the participant count has exceeded this
+// number.
+export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
+export const CONCURRENT_AUDIO_CHANNELS = 2;
+
+export function CallEventAudioRenderer({
+  vm,
+}: {
+  vm: CallViewModel;
+}): ReactNode {
+  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
+  const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
+  const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
+
+  useEffect(() => {
+    if (effectSoundVolume === 0) {
+      return;
+    }
+    const joinSub = vm.memberChanges
+      .pipe(
+        filter(
+          ({ joined, ids }) =>
+            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
+        ),
+      )
+      .subscribe(({ joined }) => {
+        const availablePlayer = callEntered.current.find((v) => v?.paused);
+        void availablePlayer?.play();
+      });
+
+    const leftSub = vm.memberChanges
+      .pipe(
+        filter(
+          ({ ids, left }) =>
+            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
+        ),
+      )
+      .subscribe(() => {
+        const availablePlayer = callLeft.current.find((v) => v?.paused);
+        void availablePlayer?.play();
+      });
+
+    return (): void => {
+      joinSub.unsubscribe();
+      leftSub.unsubscribe();
+    };
+  }, [effectSoundVolume, callEntered, callLeft, vm]);
+
+  // Set volume.
+  useEffect(() => {
+    callEntered.current.forEach((a) => {
+      if (a) {
+        a.volume = effectSoundVolume;
+      }
+    });
+    callLeft.current.forEach((a) => {
+      if (a) {
+        a.volume = effectSoundVolume;
+      }
+    });
+  }, [callEntered, callLeft, effectSoundVolume]);
+
+  // Do not render any audio elements if playback is disabled. Will save
+  // audio file fetches.
+  if (effectSoundVolume === 0) {
+    return null;
+  }
+
+  return (
+    // Will play as soon as it's mounted, which is what we want as this will
+    // play when the call is entered.
+    <>
+      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
+        <audio
+          key={index}
+          ref={(r) => (callEntered.current[index] = r)}
+          preload="auto"
+          hidden
+        >
+          <source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
+          <source src={enterCallSoundMp3} type="audio/mpeg" />
+        </audio>
+      ))}
+      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
+        <audio
+          key={index}
+          ref={(r) => (callLeft.current[index] = r)}
+          preload="auto"
+          hidden
+        >
+          <source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
+          <source src={leftCallSoundMp3} type="audio/mpeg" />
+        </audio>
+      ))}
+    </>
+  );
+}

src/room/InCallView.tsx

@@ -87,6 +87,7 @@ import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
 import { useSwitchCamera } from "./useSwitchCamera";
 import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
 import { ReactionsOverlay } from "./ReactionsOverlay";
+import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -670,6 +671,7 @@ export const InCallView: FC<InCallViewProps> = ({
         ))}
       <RoomAudioRenderer />
       {renderContent()}
+      <CallEventAudioRenderer vm={vm} />
       <audio ref={handRaisePlayer} preload="auto" hidden>
         <source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
         <source src={handSoundMp3} type="audio/mpeg" />

src/room/ReactionAudioRenderer.test.tsx

@@ -21,6 +21,7 @@ import {
   playReactionsSound,
   soundEffectVolumeSetting,
 } from "../settings/settings";
+import { mockMediaPlay } from "../utils/test";
 
 const memberUserIdAlice = "@alice:example.org";
 const memberUserIdBob = "@bob:example.org";
@@ -80,11 +81,7 @@ test("loads no audio elements when disabled in settings", () => {
 });
 
 test("will play an audio sound when there is a reaction", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -105,11 +102,7 @@ test("will play an audio sound when there is a reaction", () => {
 });
 
 test("will play the generic audio sound when there is soundless reaction", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -152,11 +145,7 @@ test("will play an audio sound with the correct volume", () => {
 });
 
 test("will play multiple audio sounds when there are multiple different reactions", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
 
   const room = new MockRoom(memberUserIdAlice);

src/sound/LICENCE.md

@@ -19,4 +19,6 @@ The following sound effects have been originally created by Element.
 - `end_talk`
 - `start_talk_local`
 - `start_talk_remote`
+- `join_call`
+- `end_call`
 - `reactions/rock`

src/state/CallViewModel.ts

@@ -475,6 +475,19 @@ export class CallViewModel extends ViewModel {
     ),
   );
 
+  public readonly memberChanges = this.userMedia
+    .pipe(map((mediaItems) => mediaItems.map((m) => m.id)))
+    .pipe(
+      scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
+        (prev, ids) => {
+          const left = prev.ids.filter((id) => !ids.includes(id));
+          const joined = ids.filter((id) => !prev.ids.includes(id));
+          return { ids, joined, left };
+        },
+        { ids: [], joined: [], left: [] },
+      ),
+    );
+
   private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
     this.mediaItems.pipe(
       map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),

src/utils/test.ts

@@ -14,7 +14,9 @@ import {
   RemoteParticipant,
   RemoteTrackPublication,
   Room as LivekitRoom,
+  RoomEvent,
 } from "livekit-client";
+import { EventEmitter } from "stream";
 
 import {
   LocalUserMediaViewModel,
@@ -109,6 +111,33 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
   return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
 }
 
+/**
+ * A mock of a Livekit Room that can emit events.
+ */
+export class EmittableMockLivekitRoom extends EventEmitter {
+  public localParticipant?: LocalParticipant;
+  public remoteParticipants: Map<string, RemoteParticipant>;
+
+  public constructor(room: {
+    localParticipant?: LocalParticipant;
+    remoteParticipants: Map<string, RemoteParticipant>;
+  }) {
+    super();
+    this.localParticipant = room.localParticipant;
+    this.remoteParticipants = room.remoteParticipants ?? new Map();
+  }
+
+  public addParticipant(remoteParticipant: RemoteParticipant): void {
+    this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
+    this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
+  }
+
+  public removeParticipant(remoteParticipant: RemoteParticipant): void {
+    this.remoteParticipants.delete(remoteParticipant.identity);
+    this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
+  }
+}
+
 export function mockLivekitRoom(
   room: Partial<LivekitRoom>,
   {
@@ -206,3 +235,12 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
     ...config,
   });
 }
+
+export function mockMediaPlay(): string[] {
+  const audioIsPlaying: string[] = [];
+  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
+    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
+    return Promise.resolve();
+  };
+  return audioIsPlaying;
+}