studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

temp refactored membership rtcidentity

Browse Source
This commit is contained in:
Timo K
2025-12-19 19:23:41 +01:00
parent 50f3bf00ae
commit 55d18f10fe
8 changed files with 87 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -109,7 +109,7 @@
"livekit-client": "^2.13.0", "livekit-client": "^2.13.0",
"lodash-es": "^4.17.21", "lodash-es": "^4.17.21",
"loglevel": "^1.9.1", "loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725",
"matrix-widget-api": "^1.14.0", "matrix-widget-api": "^1.14.0",
"normalize.css": "^8.0.1", "normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3", "observable-hooks": "^4.2.3",

110
src/e2ee/matrixKeyProvider.ts
View File

@@ -6,18 +6,13 @@ Please see LICENSE in the repository root for full details.
*/ */
import { BaseKeyProvider } from "livekit-client"; import { BaseKeyProvider } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { import {
type MatrixRTCSession, type MatrixRTCSession,
MatrixRTCSessionEvent, MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc"; } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { firstValueFrom } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
const logger = rootLogger.getChild("[MatrixKeyProvider]");
import {
computeLivekitParticipantIdentity$,
livekitIdentityInput,
} from "../state/CallViewModel/remoteMembers/LivekitParticipantIdentity";
export class MatrixKeyProvider extends BaseKeyProvider { export class MatrixKeyProvider extends BaseKeyProvider {
private rtcSession?: MatrixRTCSession; private rtcSession?: MatrixRTCSession;
@@ -32,6 +27,10 @@ export class MatrixKeyProvider extends BaseKeyProvider {
MatrixRTCSessionEvent.EncryptionKeyChanged, MatrixRTCSessionEvent.EncryptionKeyChanged,
this.onEncryptionKeyChanged, this.onEncryptionKeyChanged,
); );
this.rtcSession.off(
MatrixRTCSessionEvent.MembershipsChanged,
this.onMembershipsChanged,
);
} }
this.rtcSession = rtcSession; this.rtcSession = rtcSession;
@@ -40,55 +39,86 @@ export class MatrixKeyProvider extends BaseKeyProvider {
MatrixRTCSessionEvent.EncryptionKeyChanged, MatrixRTCSessionEvent.EncryptionKeyChanged,
this.onEncryptionKeyChanged, this.onEncryptionKeyChanged,
); );
this.rtcSession.on(
MatrixRTCSessionEvent.MembershipsChanged,
this.onMembershipsChanged,
);
// The new session could be aware of keys of which the old session wasn't, // The new session could be aware of keys of which the old session wasn't,
// so emit key changed events // so emit key changed events
this.rtcSession.reemitEncryptionKeys(); this.rtcSession.reemitEncryptionKeys();
} }
private keyCache = new Array<{
membership: CallMembershipIdentityParts;
encryptionKey: Uint8Array;
encryptionKeyIndex: number;
}>();
private onMembershipsChanged = (): void => {
const duplicatedArray = this.keyCache;
// Reset key cache first. It will get repopulated when calling `onEncryptionKeyChanged`
this.keyCache = [];
let next = duplicatedArray.pop();
while (next !== undefined) {
logger.debug(
"[KeyCache] remove key event from the cache and try adding it again. For membership: ",
next.membership,
);
this.onEncryptionKeyChanged(
next.encryptionKey,
next.encryptionKeyIndex,
next.membership,
);
next = duplicatedArray.pop();
}
};
private onEncryptionKeyChanged = ( private onEncryptionKeyChanged = (
encryptionKey: Uint8Array, encryptionKey: Uint8Array,
encryptionKeyIndex: number, encryptionKeyIndex: number,
membership: CallMembershipIdentityParts, membership: CallMembershipIdentityParts,
): void => { ): void => {
const unhashedIdentity = livekitIdentityInput(membership);
// This is the only way we can get the kind of the membership event we just received the key for. // This is the only way we can get the kind of the membership event we just received the key for.
// best case we want to recompute this once the memberships change (you can receive the key before the participant...) // best case we want to recompute this once the memberships change (you can receive the key before the participant...)
// const membershipFull = this.rtcSession?.memberships.find(
// TODO change this to `?? "rtc"` for newer versions. (m) =>
const kind = m.userId === membership.userId &&
this.rtcSession?.memberships.find( m.deviceId === membership.deviceId &&
(m) => m.memberId === membership.memberId,
m.userId === membership.userId && );
m.deviceId === membership.deviceId && if (!membershipFull) {
m.memberId === membership.memberId, logger.debug(
)?.kind ?? "session"; "[KeyCache] Added key event to the cache because we do not have a membership for it (yet): ",
membership,
);
this.keyCache.push({ membership, encryptionKey, encryptionKeyIndex });
return;
}
Promise.all([ crypto.subtle
crypto.subtle.importKey("raw", encryptionKey, "HKDF", false, [ .importKey("raw", encryptionKey, "HKDF", false, [
"deriveBits", "deriveBits",
"deriveKey", "deriveKey",
]), ])
firstValueFrom(computeLivekitParticipantIdentity$(membership, kind)), .then(
]).then( (keyMaterial) => {
([keyMaterial, livekitParticipantId]) => { this.onSetEncryptionKey(
this.onSetEncryptionKey( keyMaterial,
keyMaterial, membershipFull.rtcBackendIdentity,
livekitParticipantId, encryptionKeyIndex,
encryptionKeyIndex, );
);
logger.debug( logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${livekitParticipantId} (before hash: ${unhashedIdentity}) encryptionKeyIndex=${encryptionKeyIndex}`, `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${membershipFull.rtcBackendIdentity} (before hash: ${membershipFull.userId}) encryptionKeyIndex=${encryptionKeyIndex}`,
); );
}, },
(e) => { (e) => {
logger.error( logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${unhashedIdentity} encryptionKeyIndex=${encryptionKeyIndex}`, `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipFull.userId} encryptionKeyIndex=${encryptionKeyIndex}`,
e, e,
); );
}, },
); );
}; };
} }

9
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -44,7 +44,6 @@ import {
mockRtcMembership, mockRtcMembership,
testScope, testScope,
exampleTransport, exampleTransport,
mockComputeLivekitParticipantIdentity$,
} from "../../utils/test.ts"; } from "../../utils/test.ts";
import { E2eeType } from "../../e2ee/e2eeType.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts";
import { import {
@@ -88,14 +87,6 @@ vi.mock(
}), }),
); );
vi.mock(
import("./remoteMembers/LivekitParticipantIdentity.ts"),
async (importOriginal) => ({
...(await importOriginal()),
computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$,
}),
);
const yesNo = { const yesNo = {
y: true, y: true,
n: false, n: false,

53
src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts
View File

@@ -1,53 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { encodeUnpaddedBase64Url } from "matrix-js-sdk";
import { sha256 } from "matrix-js-sdk/lib/digest";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { from, type Observable } from "rxjs";
const livekitParticipantIdentityCache = new Map<string, string>();
/**
* The string that is computed based on the membership and used for the computing the hash.
* `${userId}:${deviceId}:${membershipID}`
* as the direct imput for: await sha256(input)
*/
export const livekitIdentityInput = ({
userId,
deviceId,
memberId,
}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`;
export function computeLivekitParticipantIdentity$(
membership: CallMembershipIdentityParts,
kind: "rtc" | "session",
): Observable<string> {
const compute = async (): Promise<string> => {
switch (kind) {
case "rtc": {
const input = livekitIdentityInput(membership);
if (livekitParticipantIdentityCache.size > 400)
// prevent memory leaks in a stupid/simple way
livekitParticipantIdentityCache.clear();
// TODO use non deprecated memberId
if (livekitParticipantIdentityCache.has(input))
return livekitParticipantIdentityCache.get(input)!;
else {
const hashBuffer = await sha256(input);
const hashedString = encodeUnpaddedBase64Url(hashBuffer);
livekitParticipantIdentityCache.set(input, hashedString);
return hashedString;
}
}
case "session":
default:
return `${membership.userId}:${membership.deviceId}`;
}
};
return from(compute());
}

57
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -10,7 +10,7 @@ import {
type LivekitTransport, type LivekitTransport,
type CallMembership, type CallMembership,
} from "matrix-js-sdk/lib/matrixrtc"; } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, filter, map, switchMap } from "rxjs"; import { combineLatest, filter, map } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../../Behavior"; import { type Behavior } from "../../Behavior";
@@ -18,7 +18,6 @@ import { type IConnectionManager } from "./ConnectionManager";
import { Epoch, type ObservableScope } from "../../ObservableScope"; import { Epoch, type ObservableScope } from "../../ObservableScope";
import { type Connection } from "./Connection"; import { type Connection } from "./Connection";
import { generateItemsWithEpoch } from "../../../utils/observable"; import { generateItemsWithEpoch } from "../../../utils/observable";
import { computeLivekitParticipantIdentity$ } from "./LivekitParticipantIdentity";
const logger = rootLogger.getChild("[MatrixLivekitMembers]"); const logger = rootLogger.getChild("[MatrixLivekitMembers]");
@@ -82,45 +81,12 @@ export function createMatrixLivekitMembers$({
membershipsWithTransport$, membershipsWithTransport$,
connectionManager, connectionManager,
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> { }: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
/**
* This internal observable is used to compute the async sha256 hash of the user's identity.
* a promise is treated like an observable. So we can switchMap on the promise from the identity computation.
* The last update to `membershipsWithTransport$` will always be the last promise we pass to switchMap.
* So we will eventually always end up with the latest memberships and their identities.
*/
const membershipsWithTransportAndLivekitIdentity$ =
membershipsWithTransport$.pipe(
switchMap((membershipsWithTransport) => {
const { value, epoch } = membershipsWithTransport;
const membershipsWithTransportAndLkIdentityPromises = value.map(
(obj) => {
return computeLivekitParticipantIdentity$(
obj.membership,
obj.membership.kind,
);
},
);
return combineLatest(
membershipsWithTransportAndLkIdentityPromises,
).pipe(
map((identities) => {
const membershipsWithTransportAndLkIdentity = value.map(
({ transport, membership }, index) => {
return { transport, membership, identity: identities[index] };
},
);
return new Epoch(membershipsWithTransportAndLkIdentity, epoch);
}),
);
}),
);
/** /**
* Stream of all the call members and their associated livekit data (if available). * Stream of all the call members and their associated livekit data (if available).
*/ */
return scope.behavior( return scope.behavior(
combineLatest([ combineLatest([
membershipsWithTransportAndLivekitIdentity$, membershipsWithTransport$,
connectionManager.connectionManagerData$, connectionManager.connectionManagerData$,
]).pipe( ]).pipe(
filter((values) => filter((values) =>
@@ -131,37 +97,34 @@ export function createMatrixLivekitMembers$({
// Generator function. // Generator function.
// creates an array of `{key, data}[]` // creates an array of `{key, data}[]`
// Each change in the keys (new key, missing key) will result in a call to the factory function. // Each change in the keys (new key, missing key) will result in a call to the factory function.
function* ([membershipsWithTransportAndLivekitIdentity, managerData]) { function* ([membershipsWithTransport, managerData]) {
for (const { for (const { membership, transport } of membershipsWithTransport) {
membership,
transport,
identity,
} of membershipsWithTransportAndLivekitIdentity) {
const participants = transport const participants = transport
? managerData.getParticipantForTransport(transport) ? managerData.getParticipantForTransport(transport)
: []; : [];
const participant = const participant =
participants.find((p) => p.identity == identity) ?? null; participants.find(
(p) => p.identity == membership.rtcBackendIdentity,
) ?? null;
const connection = transport const connection = transport
? managerData.getConnectionForTransport(transport) ? managerData.getConnectionForTransport(transport)
: null; : null;
yield { yield {
keys: [identity, membership.userId, membership.deviceId], keys: [membership.userId, membership.deviceId],
data: { membership, participant, connection }, data: { membership, participant, connection },
}; };
} }
}, },
// Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
(scope, data$, identity, userId, deviceId) => { (scope, data$, userId, deviceId) => {
logger.debug( logger.debug(
`Generating member for livekitIdentity: ${identity}, userId:deviceId: ${userId}${deviceId}`, `Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`,
); );
const { participant$, ...rest } = scope.splitBehavior(data$); const { participant$, ...rest } = scope.splitBehavior(data$);
// will only get called once per `participantId, userId` pair. // will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return { return {
identity,
userId, userId,
participant: { type: "remote" as const, value$: participant$ }, participant: { type: "remote" as const, value$: participant$ },
...rest, ...rest,

6
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -22,7 +22,6 @@ import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { import {
mockCallMembership, mockCallMembership,
mockComputeLivekitParticipantIdentity$,
mockMediaDevices, mockMediaDevices,
ownMemberMock, ownMemberMock,
withTestScheduler, withTestScheduler,
@@ -45,11 +44,6 @@ let lkRoomFactory: () => LivekitRoom;
const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map(); const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map();
vi.mock(import("./LivekitParticipantIdentity.ts"), async (importOriginal) => ({
...(await importOriginal()),
computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$,
}));
beforeEach(() => { beforeEach(() => {
testScope = new ObservableScope(); testScope = new ObservableScope();
mockClient = { mockClient = {

3
src/utils/test.ts
View File

@@ -252,7 +252,8 @@ export function mockRtcMembership(
content: data, content: data,
}); });
const cms = new CallMembership(event, data); const membershipData = CallMembership.membershipDataFromMatrixEvent(event);
const cms = new CallMembership(event, membershipData, "xx");
vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]);
return cms; return cms;
} }

8
yarn.lock
View File

@@ -7533,7 +7533,7 @@ __metadata:
livekit-client: "npm:^2.13.0" livekit-client: "npm:^2.13.0"
lodash-es: "npm:^4.17.21" lodash-es: "npm:^4.17.21"
loglevel: "npm:^1.9.1" loglevel: "npm:^1.9.1"
matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725"
matrix-widget-api: "npm:^1.14.0" matrix-widget-api: "npm:^1.14.0"
normalize.css: "npm:^8.0.1" normalize.css: "npm:^8.0.1"
observable-hooks: "npm:^4.2.3" observable-hooks: "npm:^4.2.3"
@@ -10338,9 +10338,9 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4": "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725":
version: 39.3.0 version: 39.3.0
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725"
dependencies: dependencies:
"@babel/runtime": "npm:^7.12.5" "@babel/runtime": "npm:^7.12.5"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0"
@@ -10356,7 +10356,7 @@ __metadata:
sdp-transform: "npm:^3.0.0" sdp-transform: "npm:^3.0.0"
unhomoglyph: "npm:^1.0.6" unhomoglyph: "npm:^1.0.6"
uuid: "npm:13" uuid: "npm:13"
checksum: 10c0/78c27847b58c229513bd28c4c4ad391d8af6722711d3d0f42e93a537d7a827a7233e920936dd8d7005c7893bad17a503c3f62b56ecfed3cf4ae81a5097b4ac21 checksum: 10c0/2e7061f6e648c91aaeb30b3e01626d855e24efcb330bbe432fcba199bd46b0b0d998cbc545748e1c72a7b643d25581f988fcad9bbaa42912a6ec96a27c41d0de
languageName: node languageName: node
linkType: hard linkType: hard