studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor reactions / hand raised to use rxjs and start ordering tiles based on hand raised. (#2885)

Browse Source 
* Add support for using CallViewModel for reactions sounds.

* Drop setting

* Convert reaction sounds to call view model / rxjs

* Use call view model for hand raised reactions

* Support raising reactions for matrix rtc members.

* Tie up last bits of useReactions

* linting

* Update calleventaudiorenderer

* Update reaction audio renderer

* more test bits

* All the test bits and pieces

* More refactors

* Refactor reactions into a sender and receiver.

* Fixup reaction toggle button

* Adapt reactions test

* Tests all pass.

* lint

* fix a couple of bugs

* remove unused helper file

* lint

* finnish notation

* Add tests for useReactionsReader

* remove mistaken vitest file

* fix

* filter

* invert

* fixup tests with fake timers

* Port useReactionsReader hook to ReactionsReader class.

* lint

* exclude some files from coverage

* Add screen share sound effect.

* cancel sub on destroy

* tidy tidy
This commit is contained in:
Will Hunt
2024-12-19 15:54:28 +00:00
committed by GitHub
parent 7d00f85abc
commit abf2ecd521
28 changed files with 1835 additions and 1184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/state/CallViewModel.test.ts
View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { test, vi, onTestFinished, it } from "vitest";
import {
BehaviorSubject,
combineLatest,
debounceTime,
distinctUntilChanged,
@@ -46,6 +47,7 @@ import {
type ECConnectionState,
} from "../livekit/useECConnectionState";
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import { showNonMemberTiles } from "../settings/settings";
vi.mock("@livekit/components-core");
@@ -190,7 +192,10 @@ function withCallViewModel(
rtcMembers$: Observable<Partial<CallMembership>[]>,
connectionState$: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
continuation: (
vm: CallViewModel,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
) => void,
): void {
const room = mockMatrixRoom({
client: {
@@ -235,6 +240,8 @@ function withCallViewModel(
{ remoteParticipants$ },
);
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
@@ -242,6 +249,8 @@ function withCallViewModel(
kind: E2eeType.PER_PARTICIPANT,
},
connectionState$,
raisedHands$,
new BehaviorSubject({}),
);
onTestFinished(() => {
@@ -252,7 +261,7 @@ function withCallViewModel(
roomEventSelectorSpy!.mockRestore();
});
continuation(vm);
continuation(vm, { raisedHands$: raisedHands$ });
}
test("participants are retained during a focus switch", () => {
@@ -782,3 +791,62 @@ it("should show at least one tile per MatrixRTCSession", () => {
);
});
});
it("should rank raised hands above video feeds and below speakers and presenters", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// There should always be one tile for each MatrixRTCSession
const expectedLayoutMarbles = "ab";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm, { raisedHands$ }) => {
schedule("ab", {
a: () => {
// We imagine that only two tiles (the first two) will be visible on screen at a time
vm.layout$.subscribe((layout) => {
if (layout.type === "grid") {
layout.setVisibleTiles(2);
}
});
},
b: () => {
raisedHands$.next({
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
time: new Date(),
reactionEventId: "",
membershipEventId: "",
},
});
},
});
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [
"local:0",
"@alice:example.org:AAAA:0",
"@bob:example.org:BBBB:0",
],
},
b: {
type: "grid",
spotlight: undefined,
grid: [
"local:0",
// Bob shifts up!
"@bob:example.org:BBBB:0",
"@alice:example.org:AAAA:0",
],
},
},
);
},
);
});
});

113
src/state/CallViewModel.ts
View File

@@ -69,7 +69,12 @@ import {
} from "./MediaViewModel";
import { accumulate, finalizeValue } from "../utils/observable";
import { ObservableScope } from "./ObservableScope";
import { duplicateTiles, showNonMemberTiles } from "../settings/settings";
import {
duplicateTiles,
playReactionsSound,
showReactions,
showNonMemberTiles,
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled$ } from "../controls";
import {
@@ -82,6 +87,11 @@ import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
type RaisedHandInfo,
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
@@ -210,6 +220,10 @@ enum SortingBin {
* Participants that have been speaking recently.
*/
Speakers,
/**
* Participants that have their hand raised.
*/
HandRaised,
/**
* Participants with video.
*/
@@ -244,6 +258,8 @@ class UserMedia {
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
this.participant$ = new BehaviorSubject(participant);
@@ -254,6 +270,8 @@ class UserMedia {
this.participant$.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
} else {
this.vm = new RemoteUserMediaViewModel(
@@ -264,6 +282,8 @@ class UserMedia {
>,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
}
@@ -473,6 +493,8 @@ export class CallViewModel extends ViewModel {
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
let participant:
| LocalParticipant
| RemoteParticipant
@@ -522,6 +544,12 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
this.handsRaised$.pipe(
map((v) => v[matrixIdentifier]?.time ?? null),
),
this.reactions$.pipe(
map((v) => v[matrixIdentifier] ?? undefined),
),
),
];
@@ -574,6 +602,8 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
of(null),
of(null),
),
];
}
@@ -681,11 +711,12 @@ export class CallViewModel extends ViewModel {
m.speaker$,
m.presenter$,
m.vm.videoEnabled$,
m.vm.handRaised$,
m.vm instanceof LocalUserMediaViewModel
? m.vm.alwaysShow$
: of(false),
],
(speaker, presenter, video, alwaysShow) => {
(speaker, presenter, video, handRaised, alwaysShow) => {
let bin: SortingBin;
if (m.vm.local)
bin = alwaysShow
@@ -693,6 +724,7 @@ export class CallViewModel extends ViewModel {
: SortingBin.SelfNotAlwaysShown;
else if (presenter) bin = SortingBin.Presenters;
else if (speaker) bin = SortingBin.Speakers;
else if (handRaised) bin = SortingBin.HandRaised;
else if (video) bin = SortingBin.Video;
else bin = SortingBin.NoVideo;
@@ -1170,6 +1202,77 @@ export class CallViewModel extends ViewModel {
}),
this.scope.state(),
);
public readonly reactions$ = this.reactionsSubject$.pipe(
map((v) =>
Object.fromEntries(
Object.entries(v).map(([a, { reactionOption }]) => [a, reactionOption]),
),
),
);
public readonly handsRaised$ = this.handsRaisedSubject$.pipe();
/**
* Emits an array of reactions that should be visible on the screen.
*/
public readonly visibleReactions$ = showReactions.value$.pipe(
switchMap((show) => (show ? this.reactions$ : of({}))),
scan<
Record<string, ReactionOption>,
{ sender: string; emoji: string; startX: number }[]
>((acc, latest) => {
const newSet: { sender: string; emoji: string; startX: number }[] = [];
for (const [sender, reaction] of Object.entries(latest)) {
const startX =
acc.find((v) => v.sender === sender && v.emoji)?.startX ??
Math.ceil(Math.random() * 80) + 10;
newSet.push({ sender, emoji: reaction.emoji, startX });
}
return newSet;
}, []),
);
/**
* Emits an array of reactions that should be played.
*/
public readonly audibleReactions$ = playReactionsSound.value$.pipe(
switchMap((show) =>
show ? this.reactions$ : of<Record<string, ReactionOption>>({}),
),
map((reactions) => Object.values(reactions).map((v) => v.name)),
scan<string[], { playing: string[]; newSounds: string[] }>(
(acc, latest) => {
return {
playing: latest.filter(
(v) => acc.playing.includes(v) || acc.newSounds.includes(v),
),
newSounds: latest.filter(
(v) => !acc.playing.includes(v) && !acc.newSounds.includes(v),
),
};
},
{ playing: [], newSounds: [] },
),
map((v) => v.newSounds),
);
/**
* Emits an event every time a new hand is raised in
* the call.
*/
public readonly newHandRaised$ = this.handsRaised$.pipe(
map((v) => Object.keys(v).length),
scan(
(acc, newValue) => ({
value: newValue,
playSounds: newValue > acc.value,
}),
{ value: 0, playSounds: false },
),
filter((v) => v.playSounds),
);
/**
* Emits an event every time a new screenshare is started in
* the call.
@@ -1192,6 +1295,12 @@ export class CallViewModel extends ViewModel {
private readonly livekitRoom: LivekitRoom,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState$: Observable<ECConnectionState>,
private readonly handsRaisedSubject$: Observable<
Record<string, RaisedHandInfo>
>,
private readonly reactionsSubject$: Observable<
Record<string, ReactionInfo>
>,
) {
super();
}

27
src/state/MediaViewModel.ts
View File

@@ -51,6 +51,7 @@ import { alwaysShowSelf } from "../settings/settings";
import { accumulate } from "../utils/observable";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { type ReactionOption } from "../reactions";
// TODO: Move this naming logic into the view model
export function useDisplayName(vm: MediaViewModel): string {
@@ -371,6 +372,8 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
public readonly handRaised$: Observable<Date | null>,
public readonly reaction$: Observable<ReactionOption | null>,
) {
super(
id,
@@ -437,8 +440,18 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
super(id, member, participant$, encryptionSystem, livekitRoom);
super(
id,
member,
participant$,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
}
}
@@ -498,8 +511,18 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
super(id, member, participant$, encryptionSystem, livekitRoom);
super(
id,
member,
participant$,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
// Sync the local volume with LiveKit
combineLatest([