studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

temp

Browse Source 
Signed-off-by: Timo K <toger5@hotmail.de>
This commit is contained in:
Timo K
2025-08-27 14:01:01 +02:00
parent 9b9c08ed61
commit 9011ae4e1f
8 changed files with 461 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/livekit/MatrixAudioRenderer.tsx
View File

@@ -78,6 +78,8 @@ export function MatrixAudioRenderer({
loggedInvalidIdentities.current.add(identity);
};
// TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the
// livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room.
const tracks = useTracks(
[
Track.Source.Microphone,

123
src/livekit/livekitSubscriptionRoom.ts Normal file
View File

@@ -0,0 +1,123 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState,
type E2EEManagerOptions,
ExternalE2EEKeyProvider,
LocalVideoTrack,
Room,
type RoomOptions,
} from "livekit-client";
import { useEffect, useRef } from "react";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { defaultLiveKitOptions } from "./options";
import { type SFUConfig } from "./openIDSFU";
import { type MuteStates } from "../room/MuteStates";
import { useMediaDevices } from "../MediaDevicesContext";
import {
type ECConnectionState,
useECConnectionState,
} from "./useECConnectionState";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "./TrackProcessorContext";
import { observeTrackReference$ } from "../state/MediaViewModel";
import { useUrlParams } from "../UrlParams";
import { useInitial } from "../useInitial";
import { getValue } from "../utils/observable";
import { type SelectedDevice } from "../state/MediaDevices";
interface UseLivekitResult {
livekitPublicationRoom?: Room;
connState: ECConnectionState;
}
// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices)
// This is not used! (but summarizes what we need)
export function livekitSubscriptionRoom(
rtcSession: MatrixRTCSession,
muteStates: MuteStates,
sfuConfig: SFUConfig | undefined,
e2eeSystem: EncryptionSystem,
): UseLivekitResult {
// Only ever create the room once via useInitial.
// The call can end up with multiple livekit rooms. This is the particular room in
// which this participant publishes their media.
const publicationRoom = useInitial(() => {
logger.info("[LivekitRoom] Create LiveKit room");
let e2ee: E2EEManagerOptions | undefined;
if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
logger.info("Created MatrixKeyProvider (per participant)");
e2ee = {
keyProvider: new MatrixKeyProvider(),
worker: new E2EEWorker(),
};
} else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
logger.info("Created ExternalE2EEKeyProvider (shared key)");
e2ee = {
keyProvider: new ExternalE2EEKeyProvider(),
worker: new E2EEWorker(),
};
}
const roomOptions: RoomOptions = {
...defaultLiveKitOptions,
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: getValue(devices.audioOutput.selected$)?.id,
},
e2ee,
};
// We have to create the room manually here due to a bug inside
// @livekit/components-react. JSON.stringify() is used in deps of a
// useEffect() with an argument that references itself, if E2EE is enabled
const room = new Room(roomOptions);
room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
});
return room;
});
// Setup and update the keyProvider which was create by `createRoom`
useEffect(() => {
const e2eeOptions = publicationRoom.options.e2ee;
if (
e2eeSystem.kind === E2eeType.NONE ||
!(e2eeOptions && "keyProvider" in e2eeOptions)
)
return;
if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
(e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession);
} else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
(e2eeOptions.keyProvider as ExternalE2EEKeyProvider)
.setKey(e2eeSystem.secret)
.catch((e) => {
logger.error("Failed to set shared key for E2EE", e);
});
}
}, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]);
return {
connState: connectionState,
livekitPublicationRoom: publicationRoom,
};
}

66
src/livekit/openIDSFU.ts
View File

@@ -7,12 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { useEffect, useState } from "react";
import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc";
import { useActiveLivekitFocus } from "../room/useActiveFocus";
import { useErrorBoundary } from "../useErrorBoundary";
import { FailToGetOpenIdToken } from "../utils/errors";
import { doNetworkOperationWithRetry } from "../utils/matrix";
@@ -34,38 +29,11 @@ export type OpenIDClientParts = Pick<
"getOpenIdToken" | "getDeviceId"
>;
export function useOpenIDSFU(
client: OpenIDClientParts,
rtcSession: MatrixRTCSession,
): SFUConfig | undefined {
const [sfuConfig, setSFUConfig] = useState<SFUConfig | undefined>(undefined);
const activeFocus = useActiveLivekitFocus(rtcSession);
const { showErrorBoundary } = useErrorBoundary();
useEffect(() => {
if (activeFocus) {
getSFUConfigWithOpenID(client, activeFocus).then(
(sfuConfig) => {
setSFUConfig(sfuConfig);
},
(e) => {
showErrorBoundary(new FailToGetOpenIdToken(e));
logger.error("Failed to get SFU config", e);
},
);
} else {
setSFUConfig(undefined);
}
}, [client, activeFocus, showErrorBoundary]);
return sfuConfig;
}
export async function getSFUConfigWithOpenID(
client: OpenIDClientParts,
activeFocus: LivekitFocus,
): Promise<SFUConfig | undefined> {
serviceUrl: string,
livekitAlias: string,
): Promise<SFUConfig> {
let openIdToken: IOpenIDToken;
try {
openIdToken = await doNetworkOperationWithRetry(async () =>
@@ -78,26 +46,16 @@ export async function getSFUConfigWithOpenID(
}
logger.debug("Got openID token", openIdToken);
try {
logger.info(
`Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...`,
);
const sfuConfig = await getLiveKitJWT(
client,
activeFocus.livekit_service_url,
activeFocus.livekit_alias,
openIdToken,
);
logger.info(`Got JWT from call's active focus URL.`);
logger.info(`Trying to get JWT for focus ${serviceUrl}...`);
const sfuConfig = await getLiveKitJWT(
client,
serviceUrl,
livekitAlias,
openIdToken,
);
logger.info(`Got JWT from call's active focus URL.`);
return sfuConfig;
} catch (e) {
logger.warn(
`Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`,
e,
);
return undefined;
}
return sfuConfig;
}
async function getLiveKitJWT(

49
src/livekit/useLivekit.ts
View File

@@ -50,11 +50,12 @@ import { getValue } from "../utils/observable";
import { type SelectedDevice } from "../state/MediaDevices";
interface UseLivekitResult {
livekitRoom?: Room;
livekitPublicationRoom?: Room;
connState: ECConnectionState;
}
export function useLivekit(
// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object.
export function useLivekitPublicationRoom(
rtcSession: MatrixRTCSession,
muteStates: MuteStates,
sfuConfig: SFUConfig | undefined,
@@ -83,7 +84,9 @@ export function useLivekit(
const { processor } = useTrackProcessor();
// Only ever create the room once via useInitial.
const room = useInitial(() => {
// The call can end up with multiple livekit rooms. This is the particular room in
// which this participant publishes their media.
const publicationRoom = useInitial(() => {
logger.info("[LivekitRoom] Create LiveKit room");
let e2ee: E2EEManagerOptions | undefined;
@@ -135,7 +138,7 @@ export function useLivekit(
// Setup and update the keyProvider which was create by `createRoom`
useEffect(() => {
const e2eeOptions = room.options.e2ee;
const e2eeOptions = publicationRoom.options.e2ee;
if (
e2eeSystem.kind === E2eeType.NONE ||
!(e2eeOptions && "keyProvider" in e2eeOptions)
@@ -151,7 +154,7 @@ export function useLivekit(
logger.error("Failed to set shared key for E2EE", e);
});
}
}, [room.options.e2ee, e2eeSystem, rtcSession]);
}, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]);
// Sync the requested track processors with LiveKit
useTrackProcessorSync(
@@ -170,7 +173,7 @@ export function useLivekit(
return track instanceof LocalVideoTrack ? track : null;
}),
),
[room],
[publicationRoom],
),
),
);
@@ -178,7 +181,7 @@ export function useLivekit(
const connectionState = useECConnectionState(
initialAudioInputId,
initialMuteStates.audio.enabled,
room,
publicationRoom,
sfuConfig,
);
@@ -216,8 +219,11 @@ export function useLivekit(
// It's important that we only do this in the connected state, because
// LiveKit's internal mute states aren't consistent during connection setup,
// and setting tracks to be enabled during this time causes errors.
if (room !== undefined && connectionState === ConnectionState.Connected) {
const participant = room.localParticipant;
if (
publicationRoom !== undefined &&
connectionState === ConnectionState.Connected
) {
const participant = publicationRoom.localParticipant;
// Always update the muteButtonState Ref so that we can read the current
// state in awaited blocks.
buttonEnabled.current = {
@@ -275,7 +281,7 @@ export function useLivekit(
audioMuteUpdating.current = true;
trackPublication = await participant.setMicrophoneEnabled(
buttonEnabled.current.audio,
room.options.audioCaptureDefaults,
publicationRoom.options.audioCaptureDefaults,
);
audioMuteUpdating.current = false;
break;
@@ -283,7 +289,7 @@ export function useLivekit(
videoMuteUpdating.current = true;
trackPublication = await participant.setCameraEnabled(
buttonEnabled.current.video,
room.options.videoCaptureDefaults,
publicationRoom.options.videoCaptureDefaults,
);
videoMuteUpdating.current = false;
break;
@@ -347,11 +353,14 @@ export function useLivekit(
logger.error("Failed to sync video mute state with LiveKit", e);
});
}
}, [room, muteStates, connectionState]);
}, [publicationRoom, muteStates, connectionState]);
useEffect(() => {
// Sync the requested devices with LiveKit's devices
if (room !== undefined && connectionState === ConnectionState.Connected) {
if (
publicationRoom !== undefined &&
connectionState === ConnectionState.Connected
) {
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
@@ -359,15 +368,15 @@ export function useLivekit(
selected$.subscribe((device) => {
logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
room.getActiveDevice(kind),
publicationRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
room.getActiveDevice(kind) !== device.id
publicationRoom.getActiveDevice(kind) !== device.id
) {
room
publicationRoom
.switchActiveDevice(kind, device.id)
.catch((e) =>
logger.error(`Failed to sync ${kind} device with LiveKit`, e),
@@ -393,7 +402,7 @@ export function useLivekit(
.pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER))
.subscribe(() => {
const activeMicTrack = Array.from(
room.localParticipant.audioTrackPublications.values(),
publicationRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
@@ -408,7 +417,7 @@ export function useLivekit(
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
room.localParticipant
publicationRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
@@ -422,10 +431,10 @@ export function useLivekit(
for (const s of subscriptions) s?.unsubscribe();
};
}
}, [room, devices, connectionState, controlledAudioDevices]);
}, [publicationRoom, devices, connectionState, controlledAudioDevices]);
return {
connState: connectionState,
livekitRoom: room,
livekitPublicationRoom: publicationRoom,
};
}