studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disambiguate displaynames (#2918)

Browse Source 
* Disambigute displaynames

* Add test

* fixup test functions

* prettier

* lint

* Split displayname utils into own file and add tests.

* Split out fixtures

* Add more testcases for displayname calculation.

* lint

* Also listen for displayname changes. (I stand corrected!)

* fix missing media tiles on missing member
This commit is contained in:
Will Hunt
2025-01-14 14:46:39 +00:00
committed by GitHub
parent d9e0c67315
commit 0f2e67dd60
9 changed files with 482 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
src/state/CallViewModel.test.ts
View File

@@ -14,6 +14,7 @@ import {
map,
type Observable,
of,
skip,
switchMap,
} from "rxjs";
import { type MatrixClient } from "matrix-js-sdk/src/matrix";
@@ -49,25 +50,39 @@ import {
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import { showNonMemberTiles } from "../settings/settings";
import {
alice,
aliceDoppelganger,
aliceDoppelgangerId,
aliceDoppelgangerRtcMember,
aliceId,
aliceParticipant,
aliceRtcMember,
bob,
bobId,
bobRtcMember,
bobZeroWidthSpace,
bobZeroWidthSpaceId,
bobZeroWidthSpaceRtcMember,
daveRTL,
daveRTLId,
daveRTLRtcMember,
local,
localId,
localRtcMember,
} from "../utils/test-fixtures";
vi.mock("@livekit/components-core");
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bob = mockMatrixRoomMember(bobRtcMember);
const carol = mockMatrixRoomMember(localRtcMember);
const dave = mockMatrixRoomMember(daveRtcMember);
const carol = local;
const carolId = localId;
const dave = mockMatrixRoomMember(daveRtcMember, { rawDisplayName: "Dave" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const aliceSharingScreen = mockRemoteParticipant({
identity: aliceId,
isScreenShareEnabled: true,
@@ -80,7 +95,9 @@ const bobSharingScreen = mockRemoteParticipant({
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const roomMembers = new Map(
[alice, bob, carol, dave].map((p) => [p.userId, p]),
[alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map(
(p) => [p.userId, p],
),
);
export interface GridLayoutSummary {
@@ -792,6 +809,127 @@ it("should show at least one tile per MatrixRTCSession", () => {
});
});
test("should disambiguate users with the same displayname", () => {
withTestScheduler(({ hot, expectObservable }) => {
const scenarioInputMarbles = "abcde";
const expectedLayoutMarbles = "abcde";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [aliceRtcMember],
c: [aliceRtcMember, aliceDoppelgangerRtcMember],
d: [aliceRtcMember, aliceDoppelgangerRtcMember, bobRtcMember],
e: [aliceDoppelgangerRtcMember, bobRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
expectedLayoutMarbles,
{
// Carol has no displayname - So userId is used.
a: new Map([[carolId, carol.userId]]),
b: new Map([
[carolId, carol.userId],
[aliceId, alice.rawDisplayName],
]),
// The second alice joins.
c: new Map([
[carolId, carol.userId],
[aliceId, "Alice (@alice:example.org)"],
[aliceDoppelgangerId, "Alice (@alice2:example.org)"],
]),
// Bob also joins
d: new Map([
[carolId, carol.userId],
[aliceId, "Alice (@alice:example.org)"],
[aliceDoppelgangerId, "Alice (@alice2:example.org)"],
[bobId, bob.rawDisplayName],
]),
// Alice leaves, and the displayname should reset.
e: new Map([
[carolId, carol.userId],
[aliceDoppelgangerId, "Alice"],
[bobId, bob.rawDisplayName],
]),
},
);
},
);
});
});
test("should disambiguate users with invisible characters", () => {
withTestScheduler(({ hot, expectObservable }) => {
const scenarioInputMarbles = "ab";
const expectedLayoutMarbles = "ab";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [bobRtcMember, bobZeroWidthSpaceRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
expectedLayoutMarbles,
{
// Carol has no displayname - So userId is used.
a: new Map([[carolId, carol.userId]]),
// Both Bobs join, and should handle zero width hacks.
b: new Map([
[carolId, carol.userId],
[bobId, `Bob (${bob.userId})`],
[bobZeroWidthSpaceId, `Bob (${bobZeroWidthSpace.userId})`],
]),
},
);
},
);
});
});
test("should strip RTL characters from displayname", () => {
withTestScheduler(({ hot, expectObservable }) => {
const scenarioInputMarbles = "ab";
const expectedLayoutMarbles = "ab";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [daveRtcMember, daveRTLRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
// Skip the null state.
expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe(
expectedLayoutMarbles,
{
// Carol has no displayname - So userId is used.
a: new Map([[carolId, carol.userId]]),
// Both Dave's join. Since after stripping
b: new Map([
[carolId, carol.userId],
// Not disambiguated
[daveId, "Dave"],
// This one is, since it's using RTL.
[daveRTLId, `evaD (${daveRTL.userId})`],
]),
},
);
},
);
});
});
it("should rank raised hands above video feeds and below speakers and presenters", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// There should always be one tile for each MatrixRTCSession

105
src/state/CallViewModel.ts
View File

@@ -1,5 +1,5 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2023, 2024, 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
@@ -19,7 +19,8 @@ import {
Track,
} from "livekit-client";
import {
type Room as MatrixRoom,
RoomStateEvent,
type Room,
type RoomMember,
} from "matrix-js-sdk/src/matrix";
import {
@@ -50,6 +51,7 @@ import {
} from "rxjs";
import { logger } from "matrix-js-sdk/src/logger";
import {
type CallMembership,
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
@@ -94,6 +96,7 @@ import {
} from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
// How long we wait after a focus switch before showing the real participant
// list again
@@ -258,6 +261,7 @@ class UserMedia {
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
@@ -270,6 +274,7 @@ class UserMedia {
this.participant$.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
displayname$,
handRaised$,
reaction$,
);
@@ -282,6 +287,7 @@ class UserMedia {
>,
encryptionSystem,
livekitRoom,
displayname$,
handRaised$,
reaction$,
);
@@ -333,6 +339,7 @@ class ScreenShare {
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
displayname$: Observable<string>,
) {
this.participant$ = new BehaviorSubject(participant);
@@ -342,6 +349,7 @@ class ScreenShare {
this.participant$.asObservable(),
encryptionSystem,
liveKitRoom,
displayname$,
participant.isLocal,
);
}
@@ -353,26 +361,26 @@ class ScreenShare {
type MediaItem = UserMedia | ScreenShare;
function findMatrixRoomMember(
room: MatrixRoom,
id: string,
): RoomMember | undefined {
if (id === "local")
return room.getMember(room.client.getUserId()!) ?? undefined;
function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: Room,
): { id: string; member: RoomMember | undefined } {
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let id = rtcMember.sender + ":" + rtcMember.deviceId;
const parts = id.split(":");
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
if (parts.length < 3) {
logger.warn(
`Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`,
);
return undefined;
if (!rtcMember.sender) {
return { id, member: undefined };
}
if (
rtcMember.sender === room.client.getUserId() &&
rtcMember.deviceId === room.client.getDeviceId()
) {
id = "local";
}
parts.pop();
const userId = parts.join(":");
return room.getMember(userId) ?? undefined;
const member = room.getMember(rtcMember.sender) ?? undefined;
return { id, member };
}
// TODO: Move wayyyy more business logic from the call and lobby views into here
@@ -456,6 +464,40 @@ export class CallViewModel extends ViewModel {
},
);
/**
* Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members
* joined to the call are considered here.
*/
public readonly memberDisplaynames$ = merge(
// Handle call membership changes.
fromEvent(this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged),
// Handle room membership changes (and displayname updates)
fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members),
).pipe(
startWith(null),
map(() => {
const displaynameMap = new Map<string, string>();
const { room, memberships } = this.matrixRTCSession;
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
const { member } = getRoomMemberFromRtcMember(rtcMember, room);
if (!member) {
logger.error("Could not find member for media id:", matrixIdentifier);
continue;
}
const disambiguate = shouldDisambiguate(member, memberships, room);
displaynameMap.set(
matrixIdentifier,
calculateDisplayName(member, disambiguate),
);
}
return displaynameMap;
}),
);
/**
* List of MediaItems that we want to display
*/
@@ -485,25 +527,18 @@ export class CallViewModel extends ViewModel {
) => {
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
const room = this.matrixRTCSession.room;
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
const room = this.matrixRTCSession.room;
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
const { member, id: livekitParticipantId } =
getRoomMemberFromRtcMember(rtcMember, room);
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
let participant:
| LocalParticipant
| RemoteParticipant
| undefined = undefined;
if (
rtcMember.sender === room.client.getUserId()! &&
rtcMember.deviceId === room.client.getDeviceId()
) {
livekitParticipantId = "local";
if (livekitParticipantId === "local") {
participant = localParticipant;
} else {
participant = remoteParticipants.find(
@@ -511,7 +546,6 @@ export class CallViewModel extends ViewModel {
);
}
const member = findMatrixRoomMember(room, livekitParticipantId);
if (!member) {
logger.error(
"Could not find member for media id: ",
@@ -544,6 +578,9 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
this.memberDisplaynames$.pipe(
map((m) => m.get(matrixIdentifier) ?? "[👻]"),
),
this.handsRaised$.pipe(
map((v) => v[matrixIdentifier]?.time ?? null),
),
@@ -564,6 +601,9 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
this.memberDisplaynames$.pipe(
map((m) => m.get(livekitParticipantId) ?? "[👻]"),
),
),
];
}
@@ -602,6 +642,9 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
this.memberDisplaynames$.pipe(
map((m) => m.get(participant.identity) ?? "[👻]"),
),
of(null),
of(null),
),

40
src/state/MediaViewModel.ts
View File

@@ -26,7 +26,7 @@ import {
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { type RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix";
import { type RoomMember } from "matrix-js-sdk/src/matrix";
import {
BehaviorSubject,
type Observable,
@@ -43,38 +43,15 @@ import {
switchMap,
throttleTime,
} from "rxjs";
import { useEffect } from "react";
import { ViewModel } from "./ViewModel";
import { useReactiveState } from "../useReactiveState";
import { alwaysShowSelf, showConnectionStats } from "../settings/settings";
import { alwaysShowSelf } from "../settings/settings";
import { showConnectionStats } from "../settings/settings";
import { accumulate } from "../utils/observable";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { type ReactionOption } from "../reactions";
// TODO: Move this naming logic into the view model
export function useDisplayName(vm: MediaViewModel): string {
const [displayName, setDisplayName] = useReactiveState(
() => vm.member?.rawDisplayName ?? "[👻]",
[vm.member],
);
useEffect(() => {
if (vm.member) {
const updateName = (): void => {
setDisplayName(vm.member!.rawDisplayName);
};
vm.member!.on(RoomMemberEvent.Name, updateName);
return (): void => {
vm.member!.removeListener(RoomMemberEvent.Name, updateName);
};
}
}, [vm.member, setDisplayName]);
return displayName;
}
export function observeTrackReference$(
participant$: Observable<Participant | undefined>,
source: Track.Source,
@@ -280,6 +257,7 @@ abstract class BaseMediaViewModel extends ViewModel {
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
public readonly displayname$: Observable<string>,
) {
super();
const audio$ = observeTrackReference$(participant$, audioSource).pipe(
@@ -408,6 +386,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
displayname$: Observable<string>,
public readonly handRaised$: Observable<Date | null>,
public readonly reaction$: Observable<ReactionOption | null>,
) {
@@ -419,6 +398,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
displayname$,
);
const media$ = participant$.pipe(
@@ -450,6 +430,8 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
}
/**
},
},
* The local participant's user media.
*/
export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
@@ -483,6 +465,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
@@ -492,6 +475,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
displayname$,
handRaised$,
reaction$,
);
@@ -574,6 +558,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
@@ -583,6 +568,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
displayname$,
handRaised$,
reaction$,
);
@@ -637,6 +623,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
displayname$: Observable<string>,
public readonly local: boolean,
) {
super(
@@ -647,6 +634,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
displayname$,
);
}
}