finish up most of our helper classes. there are no lint issues in the
new classes. The CallViewModel is not done yet however
This commit is contained in:
Timo K
328
src/state/localMember/LocalMembership.ts
Normal file
328
src/state/localMember/LocalMembership.ts
Normal file
@@ -0,0 +1,328 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type LocalTrack, type E2EEOptions } from "livekit-client";
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type MatrixRTCSession,
|
||||
MembershipManagerEvent,
|
||||
Status,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
ClientEvent,
|
||||
type MatrixClient,
|
||||
SyncState,
|
||||
type Room as MatrixRoom,
|
||||
} from "matrix-js-sdk";
|
||||
import {
|
||||
BehaviorSubject,
|
||||
combineLatest,
|
||||
fromEvent,
|
||||
map,
|
||||
type Observable,
|
||||
of,
|
||||
startWith,
|
||||
switchMap,
|
||||
tap,
|
||||
} from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type Behavior } from "../Behavior";
|
||||
import { type ConnectionManager } from "../remoteMembers/ConnectionManager";
|
||||
import { type ObservableScope } from "../ObservableScope";
|
||||
import { Publisher } from "./Publisher";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
|
||||
import { type MediaDevices } from "../MediaDevices";
|
||||
import { and$ } from "../../utils/observable";
|
||||
import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger";
|
||||
import {
|
||||
enterRTCSession,
|
||||
type EnterRTCSessionOptions,
|
||||
} from "../../rtcSessionHelpers";
|
||||
|
||||
/*
|
||||
* - get well known
|
||||
* - get oldest membership
|
||||
* - get transport to use
|
||||
* - get openId + jwt token
|
||||
* - wait for createTrack() call
|
||||
* - create tracks
|
||||
* - wait for join() call
|
||||
* - Publisher.publishTracks()
|
||||
* - send join state/sticky event
|
||||
*/
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
mediaDevices: MediaDevices;
|
||||
muteStates: MuteStates;
|
||||
connectionManager: ConnectionManager;
|
||||
matrixRTCSession: MatrixRTCSession;
|
||||
matrixRoom: MatrixRoom;
|
||||
localTransport$: Behavior<LivekitTransport>;
|
||||
client: MatrixClient;
|
||||
roomId: string;
|
||||
e2eeLivekitOptions: E2EEOptions | undefined;
|
||||
trackerProcessorState$: Behavior<ProcessorState>;
|
||||
}
|
||||
enum LivekitState {
|
||||
UNINITIALIZED = "uninitialized",
|
||||
CONNECTING = "connecting",
|
||||
CONNECTED = "connected",
|
||||
ERROR = "error",
|
||||
DISCONNECTED = "disconnected",
|
||||
DISCONNECTING = "disconnecting",
|
||||
}
|
||||
type LocalMemberLivekitState =
|
||||
| { state: LivekitState.ERROR; error: string }
|
||||
| { state: LivekitState.CONNECTED }
|
||||
| { state: LivekitState.CONNECTING }
|
||||
| { state: LivekitState.UNINITIALIZED }
|
||||
| { state: LivekitState.DISCONNECTED }
|
||||
| { state: LivekitState.DISCONNECTING };
|
||||
|
||||
enum MatrixState {
|
||||
CONNECTED = "connected",
|
||||
DISCONNECTED = "disconnected",
|
||||
CONNECTING = "connecting",
|
||||
}
|
||||
type LocalMemberMatrixState =
|
||||
| { state: MatrixState.CONNECTED }
|
||||
| { state: MatrixState.CONNECTING }
|
||||
| { state: MatrixState.DISCONNECTED };
|
||||
|
||||
interface LocalMemberState {
|
||||
livekit$: BehaviorSubject<LocalMemberLivekitState>;
|
||||
matrix$: BehaviorSubject<LocalMemberMatrixState>;
|
||||
}
|
||||
/**
|
||||
* This class is responsible for managing the own membership in a room.
|
||||
* We want
|
||||
* - a publisher
|
||||
* -
|
||||
* @param param0
|
||||
* @returns
|
||||
* - publisher: The handle to create tracks and publish them to the room.
|
||||
* - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation)
|
||||
* - transport$: the transport object the ownMembership$ ended up using.
|
||||
*
|
||||
*/
|
||||
export const localMembership$ = ({
|
||||
scope,
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
connectionManager,
|
||||
matrixRTCSession,
|
||||
localTransport$,
|
||||
matrixRoom,
|
||||
e2eeLivekitOptions,
|
||||
trackerProcessorState$,
|
||||
}: Props): {
|
||||
// publisher: Publisher
|
||||
requestConnect: (options: EnterRTCSessionOptions) => LocalMemberState;
|
||||
startTracks: () => Behavior<LocalTrack[]>;
|
||||
requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
|
||||
state: LocalMemberState; // TODO this is probably superseeded by joinState$
|
||||
homeserverConnected$: Behavior<boolean>;
|
||||
connected$: Behavior<boolean>;
|
||||
} => {
|
||||
const state = {
|
||||
livekit$: new BehaviorSubject<LocalMemberLivekitState>({
|
||||
state: LivekitState.UNINITIALIZED,
|
||||
}),
|
||||
matrix$: new BehaviorSubject<LocalMemberMatrixState>({
|
||||
state: MatrixState.DISCONNECTED,
|
||||
}),
|
||||
};
|
||||
|
||||
// This should be used in a combineLatest with publisher$ to connect.
|
||||
// to make it possible to call startTracks before the preferredTransport$ has resolved.
|
||||
const shouldStartTracks$ = new BehaviorSubject(false);
|
||||
|
||||
// This should be used in a combineLatest with publisher$ to connect.
|
||||
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
|
||||
|
||||
const connection$ = scope.behavior(
|
||||
combineLatest([connectionManager.connections$, localTransport$]).pipe(
|
||||
map(([connections, transport]) =>
|
||||
connections.find((connection) =>
|
||||
areLivekitTransportsEqual(connection.transport, transport),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
/**
|
||||
* Whether we are connected to the MatrixRTC session.
|
||||
*/
|
||||
const homeserverConnected$ = scope.behavior(
|
||||
// To consider ourselves connected to MatrixRTC, we check the following:
|
||||
and$(
|
||||
// The client is connected to the sync loop
|
||||
(
|
||||
fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable<
|
||||
[SyncState]
|
||||
>
|
||||
).pipe(
|
||||
startWith([matrixRoom.client.getSyncState()]),
|
||||
map(([state]) => state === SyncState.Syncing),
|
||||
),
|
||||
// Room state observed by session says we're connected
|
||||
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
|
||||
startWith(null),
|
||||
map(() => matrixRTCSession.membershipStatus === Status.Connected),
|
||||
),
|
||||
// Also watch out for warnings that we've likely hit a timeout and our
|
||||
// delayed leave event is being sent (this condition is here because it
|
||||
// provides an earlier warning than the sync loop timeout, and we wouldn't
|
||||
// see the actual leave event until we reconnect to the sync loop)
|
||||
fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe(
|
||||
startWith(null),
|
||||
map(() => matrixRTCSession.probablyLeft !== true),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// /**
|
||||
// * Whether we are "fully" connected to the call. Accounts for both the
|
||||
// * connection to the MatrixRTC session and the LiveKit publish connection.
|
||||
// */
|
||||
// // TODO use this in combination with the MemberState.
|
||||
const connected$ = scope.behavior(
|
||||
and$(
|
||||
homeserverConnected$,
|
||||
connection$.pipe(
|
||||
switchMap((c) =>
|
||||
c
|
||||
? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom"))
|
||||
: of(false),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const publisher$ = scope.behavior(
|
||||
connection$.pipe(
|
||||
map((connection) =>
|
||||
connection
|
||||
? new Publisher(
|
||||
scope,
|
||||
connection,
|
||||
mediaDevices,
|
||||
muteStates,
|
||||
e2eeLivekitOptions,
|
||||
trackerProcessorState$,
|
||||
)
|
||||
: null,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
combineLatest(
|
||||
[publisher$, shouldStartTracks$],
|
||||
(publisher, shouldStartTracks) => {
|
||||
if (publisher && shouldStartTracks) {
|
||||
publisher
|
||||
.createAndSetupTracks()
|
||||
.then((tracks) => {
|
||||
tracks$.next(tracks);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error("Error creating tracks:", error);
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// MATRIX RELATED
|
||||
|
||||
// /**
|
||||
// * Whether we should tell the user that we're reconnecting to the call.
|
||||
// */
|
||||
// // DISCUSSION own membership manager
|
||||
// const reconnecting$ = scope.behavior(
|
||||
// connected$.pipe(
|
||||
// // We are reconnecting if we previously had some successful initial
|
||||
// // connection but are now disconnected
|
||||
// scan(
|
||||
// ({ connectedPreviously }, connectedNow) => ({
|
||||
// connectedPreviously: connectedPreviously || connectedNow,
|
||||
// reconnecting: connectedPreviously && !connectedNow,
|
||||
// }),
|
||||
// { connectedPreviously: false, reconnecting: false },
|
||||
// ),
|
||||
// map(({ reconnecting }) => reconnecting),
|
||||
// ),
|
||||
// );
|
||||
|
||||
const startTracks = (): Behavior<LocalTrack[]> => {
|
||||
shouldStartTracks$.next(true);
|
||||
return tracks$;
|
||||
};
|
||||
|
||||
// const joinState$ = new BehaviorSubject<LocalMemberLivekitState>({
|
||||
// state: LivekitState.UNINITIALIZED,
|
||||
// });
|
||||
|
||||
const requestConnect = (
|
||||
options: EnterRTCSessionOptions,
|
||||
): LocalMemberState => {
|
||||
if (state.livekit$.value === null) {
|
||||
startTracks();
|
||||
state.livekit$.next({ state: LivekitState.CONNECTING });
|
||||
combineLatest([publisher$, tracks$], (publisher, tracks) => {
|
||||
publisher
|
||||
?.startPublishing()
|
||||
.then(() => {
|
||||
state.livekit$.next({ state: LivekitState.CONNECTED });
|
||||
})
|
||||
.catch((error) => {
|
||||
state.livekit$.next({ state: LivekitState.ERROR, error });
|
||||
});
|
||||
});
|
||||
}
|
||||
if (state.matrix$.value.state !== MatrixState.DISCONNECTED) {
|
||||
state.matrix$.next({ state: MatrixState.CONNECTING });
|
||||
localTransport$.pipe(
|
||||
tap((transport) => {
|
||||
enterRTCSession(matrixRTCSession, transport, options).catch(
|
||||
(error) => {
|
||||
logger.error(error);
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
|
||||
const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
|
||||
if (state.livekit$.value.state !== LivekitState.CONNECTED) return null;
|
||||
state.livekit$.next({ state: LivekitState.DISCONNECTING });
|
||||
combineLatest([publisher$, tracks$], (publisher, tracks) => {
|
||||
publisher
|
||||
?.stopPublishing()
|
||||
.then(() => {
|
||||
tracks.forEach((track) => track.stop());
|
||||
state.livekit$.next({ state: LivekitState.DISCONNECTED });
|
||||
})
|
||||
.catch((error) => {
|
||||
state.livekit$.next({ state: LivekitState.ERROR, error });
|
||||
});
|
||||
});
|
||||
|
||||
return state.livekit$;
|
||||
};
|
||||
|
||||
return {
|
||||
startTracks,
|
||||
requestConnect,
|
||||
requestDisconnect,
|
||||
state,
|
||||
homeserverConnected$,
|
||||
connected$,
|
||||
};
|
||||
};
|
||||
166
src/state/localMember/LocalTransport.ts
Normal file
166
src/state/localMember/LocalTransport.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CallMembership,
|
||||
isLivekitTransport,
|
||||
type LivekitTransportConfig,
|
||||
type LivekitTransport,
|
||||
isLivekitTransportConfig,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
|
||||
import { type Behavior } from "../Behavior.ts";
|
||||
import { type ObservableScope } from "../ObservableScope.ts";
|
||||
import { Config } from "../../config/Config.ts";
|
||||
import { MatrixRTCTransportMissingError } from "../../utils/errors.ts";
|
||||
import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts";
|
||||
|
||||
/*
|
||||
* - get well known
|
||||
* - get oldest membership
|
||||
* - get transport to use
|
||||
* - get openId + jwt token
|
||||
* - wait for createTrack() call
|
||||
* - create tracks
|
||||
* - wait for join() call
|
||||
* - Publisher.publishTracks()
|
||||
* - send join state/sticky event
|
||||
*/
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
memberships$: Behavior<CallMembership[]>;
|
||||
client: MatrixClient;
|
||||
roomId: string;
|
||||
useOldestMember$: Behavior<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
* This class is responsible for managing the local transport.
|
||||
* "Which transport is the local member going to use"
|
||||
*
|
||||
* @prop useOldestMember Whether to use the same transport as the oldest member.
|
||||
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
|
||||
*/
|
||||
export const localTransport$ = ({
|
||||
scope,
|
||||
memberships$,
|
||||
client,
|
||||
roomId,
|
||||
useOldestMember$,
|
||||
}: Props): Behavior<LivekitTransport | undefined> => {
|
||||
/**
|
||||
* The transport over which we should be actively publishing our media.
|
||||
* undefined when not joined.
|
||||
*/
|
||||
const oldestMemberTransport$ = scope.behavior(
|
||||
memberships$.pipe(
|
||||
map((memberships) => memberships[0].getTransport(memberships[0])),
|
||||
first((t) => t != undefined && isLivekitTransport(t)),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
|
||||
/**
|
||||
* The transport that we would personally prefer to publish on (if not for the
|
||||
* transport preferences of others, perhaps).
|
||||
*/
|
||||
const preferredTransport$: Behavior<LivekitTransport | undefined> =
|
||||
scope.behavior(from(makeTransport(client, roomId)), undefined);
|
||||
|
||||
/**
|
||||
* The transport we should advertise in our MatrixRTC membership (plus whether
|
||||
* it is a multi-SFU transport and whether we should use sticky events).
|
||||
*/
|
||||
const advertisedTransport$ = scope.behavior(
|
||||
combineLatest(
|
||||
[useOldestMember$, preferredTransport$, oldestMemberTransport$],
|
||||
(useOldestMember, preferredTransport, oldestMemberTransport) =>
|
||||
useOldestMember ? oldestMemberTransport : preferredTransport,
|
||||
).pipe<LivekitTransport>(distinctUntilChanged(deepCompare)),
|
||||
undefined,
|
||||
);
|
||||
return advertisedTransport$;
|
||||
};
|
||||
|
||||
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
|
||||
|
||||
async function makeTransportInternal(
|
||||
client: MatrixClient,
|
||||
roomId: string,
|
||||
): Promise<LivekitTransport> {
|
||||
logger.log("Searching for a preferred transport");
|
||||
//TODO refactor this to use the jwt service returned alias.
|
||||
const livekitAlias = roomId;
|
||||
|
||||
// TODO-MULTI-SFU: Either remove this dev tool or make it more official
|
||||
const urlFromStorage =
|
||||
localStorage.getItem("robin-matrixrtc-auth") ??
|
||||
localStorage.getItem("timo-focus-url");
|
||||
if (urlFromStorage !== null) {
|
||||
const transportFromStorage: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: urlFromStorage,
|
||||
livekit_alias: livekitAlias,
|
||||
};
|
||||
logger.log(
|
||||
"Using LiveKit transport from local storage: ",
|
||||
transportFromStorage,
|
||||
);
|
||||
return transportFromStorage;
|
||||
}
|
||||
|
||||
// Prioritize the .well-known/matrix/client, if available, over the configured SFU
|
||||
const domain = client.getDomain();
|
||||
if (domain) {
|
||||
// we use AutoDiscovery instead of relying on the MatrixClient having already
|
||||
// been fully configured and started
|
||||
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
|
||||
FOCI_WK_KEY
|
||||
];
|
||||
if (Array.isArray(wellKnownFoci)) {
|
||||
const transport: LivekitTransportConfig | undefined = wellKnownFoci.find(
|
||||
(f) => f && isLivekitTransportConfig(f),
|
||||
);
|
||||
if (transport !== undefined) {
|
||||
logger.log("Using LiveKit transport from .well-known: ", transport);
|
||||
return { ...transport, livekit_alias: livekitAlias };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const urlFromConf = Config.get().livekit?.livekit_service_url;
|
||||
if (urlFromConf) {
|
||||
const transportFromConf: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: urlFromConf,
|
||||
livekit_alias: livekitAlias,
|
||||
};
|
||||
logger.log("Using LiveKit transport from config: ", transportFromConf);
|
||||
return transportFromConf;
|
||||
}
|
||||
|
||||
throw new MatrixRTCTransportMissingError(domain ?? "");
|
||||
}
|
||||
|
||||
async function makeTransport(
|
||||
client: MatrixClient,
|
||||
roomId: string,
|
||||
): Promise<LivekitTransport> {
|
||||
const transport = await makeTransportInternal(client, roomId);
|
||||
// this will call the jwt/sfu/get endpoint to pre create the livekit room.
|
||||
await getSFUConfigWithOpenID(
|
||||
client,
|
||||
transport.livekit_service_url,
|
||||
transport.livekit_alias,
|
||||
);
|
||||
return transport;
|
||||
}
|
||||
313
src/state/localMember/Publisher.ts
Normal file
313
src/state/localMember/Publisher.ts
Normal file
@@ -0,0 +1,313 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
import {
|
||||
type E2EEOptions,
|
||||
LocalVideoTrack,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
ConnectionState as LivekitConnectionState,
|
||||
} from "livekit-client";
|
||||
import {
|
||||
map,
|
||||
NEVER,
|
||||
type Observable,
|
||||
type Subscription,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import type { Behavior } from "../Behavior.ts";
|
||||
import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts";
|
||||
import type { MuteStates } from "../MuteStates.ts";
|
||||
import {
|
||||
type ProcessorState,
|
||||
trackProcessorSync,
|
||||
} from "../../livekit/TrackProcessorContext.tsx";
|
||||
import { getUrlParams } from "../../UrlParams.ts";
|
||||
import { observeTrackReference$ } from "../MediaViewModel.ts";
|
||||
import { type Connection } from "../remoteMembers/Connection.ts";
|
||||
import { type ObservableScope } from "../ObservableScope.ts";
|
||||
|
||||
/**
|
||||
* A wrapper for a Connection object.
|
||||
* This wrapper will manage the connection used to publish to the LiveKit room.
|
||||
* The Publisher is also responsible for creating the media tracks.
|
||||
*/
|
||||
export class Publisher {
|
||||
public tracks: LocalTrack<Track.Kind>[] = [];
|
||||
/**
|
||||
* Creates a new Publisher.
|
||||
* @param scope - The observable scope to use for managing the publisher.
|
||||
* @param connection - The connection to use for publishing.
|
||||
* @param devices - The media devices to use for audio and video input.
|
||||
* @param muteStates - The mute states for audio and video.
|
||||
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
|
||||
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
|
||||
*/
|
||||
public constructor(
|
||||
private scope: ObservableScope,
|
||||
private connection: Connection,
|
||||
devices: MediaDevices,
|
||||
private readonly muteStates: MuteStates,
|
||||
e2eeLivekitOptions: E2EEOptions | undefined,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
private logger?: Logger,
|
||||
) {
|
||||
this.logger?.info("[PublishConnection] Create LiveKit room");
|
||||
const { controlledAudioDevices } = getUrlParams();
|
||||
|
||||
const room = connection.livekitRoom;
|
||||
|
||||
room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => {
|
||||
this.logger?.error("Failed to set E2EE enabled on room", e);
|
||||
});
|
||||
|
||||
// Setup track processor syncing (blur)
|
||||
this.observeTrackProcessors(scope, room, trackerProcessorState$);
|
||||
// Observe media device changes and update LiveKit active devices accordingly
|
||||
this.observeMediaDevices(scope, devices, controlledAudioDevices);
|
||||
|
||||
this.workaroundRestartAudioInputTrackChrome(devices, scope);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the connection to LiveKit and publish local tracks.
|
||||
*
|
||||
* This will:
|
||||
* wait for the connection to be ready.
|
||||
// * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
|
||||
// * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
|
||||
// * 3. Connect to the configured LiveKit room.
|
||||
// * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
|
||||
*
|
||||
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
|
||||
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
|
||||
*/
|
||||
public async createAndSetupTracks(): Promise<LocalTrack[]> {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
// Observe mute state changes and update LiveKit microphone/camera states accordingly
|
||||
this.observeMuteStates(this.scope);
|
||||
|
||||
// TODO: This should be an autostarted connection no need to start here. just check the connection state.
|
||||
// TODO: This will fetch the JWT token. Perhaps we could keep it preloaded
|
||||
// instead? This optimization would only be safe for a publish connection,
|
||||
// because we don't want to leak the user's intent to perhaps join a call to
|
||||
// remote servers before they actually commit to it.
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const sub = this.connection.state$.subscribe((s) => {
|
||||
if (s.state !== "FailedToStart") {
|
||||
reject(new Error("Disconnected from LiveKit server"));
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
sub.unsubscribe();
|
||||
}
|
||||
// TODO-MULTI-SFU: Prepublish a microphone track
|
||||
const audio = this.muteStates.audio.enabled$.value;
|
||||
const video = this.muteStates.video.enabled$.value;
|
||||
// createTracks throws if called with audio=false and video=false
|
||||
if (audio || video) {
|
||||
// TODO this can still throw errors? It will also prompt for permissions if not already granted
|
||||
this.tracks = await lkRoom.localParticipant.createTracks({
|
||||
audio,
|
||||
video,
|
||||
});
|
||||
}
|
||||
return this.tracks;
|
||||
}
|
||||
|
||||
public async startPublishing(): Promise<LocalTrack[]> {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const sub = this.connection.state$.subscribe((s) => {
|
||||
switch (s.state) {
|
||||
case "ConnectedToLkRoom":
|
||||
resolve();
|
||||
break;
|
||||
case "FailedToStart":
|
||||
reject(new Error("Failed to connect to LiveKit server"));
|
||||
break;
|
||||
default:
|
||||
this.logger?.info("waiting for connection: ", s.state);
|
||||
}
|
||||
});
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
sub.unsubscribe();
|
||||
}
|
||||
for (const track of this.tracks) {
|
||||
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
|
||||
// with a timeout.
|
||||
await lkRoom.localParticipant.publishTrack(track);
|
||||
|
||||
// TODO: check if the connection is still active? and break the loop if not?
|
||||
}
|
||||
return this.tracks;
|
||||
}
|
||||
|
||||
public async stopPublishing(): Promise<void> {
|
||||
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
|
||||
// actually has the right lifetime
|
||||
this.muteStates.audio.unsetHandler();
|
||||
this.muteStates.video.unsetHandler();
|
||||
|
||||
const localParticipant = this.connection.livekitRoom.localParticipant;
|
||||
const tracks: LocalTrack[] = [];
|
||||
const addToTracksIfDefined = (p: LocalTrackPublication): void => {
|
||||
if (p.track !== undefined) tracks.push(p.track);
|
||||
};
|
||||
localParticipant.trackPublications.forEach(addToTracksIfDefined);
|
||||
await localParticipant.unpublishTracks(tracks);
|
||||
}
|
||||
|
||||
/// Private methods
|
||||
|
||||
// Restart the audio input track whenever we detect that the active media
|
||||
// device has changed to refer to a different hardware device. We do this
|
||||
// for the sake of Chrome, which provides a "default" device that is meant
|
||||
// to match the system's default audio input, whatever that may be.
|
||||
// This is special-cased for only audio inputs because we need to dig around
|
||||
// in the LocalParticipant object for the track object and there's not a nice
|
||||
// way to do that generically. There is usually no OS-level default video capture
|
||||
// device anyway, and audio outputs work differently.
|
||||
private workaroundRestartAudioInputTrackChrome(
|
||||
devices: MediaDevices,
|
||||
scope: ObservableScope,
|
||||
): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
devices.audioInput.selected$
|
||||
.pipe(
|
||||
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
|
||||
scope.bind(),
|
||||
)
|
||||
.subscribe(() => {
|
||||
if (lkRoom.state != LivekitConnectionState.Connected) return;
|
||||
const activeMicTrack = Array.from(
|
||||
lkRoom.localParticipant.audioTrackPublications.values(),
|
||||
).find((d) => d.source === Track.Source.Microphone)?.track;
|
||||
|
||||
if (
|
||||
activeMicTrack &&
|
||||
// only restart if the stream is still running: LiveKit will detect
|
||||
// when a track stops & restart appropriately, so this is not our job.
|
||||
// Plus, we need to avoid restarting again if the track is already in
|
||||
// the process of being restarted.
|
||||
activeMicTrack.mediaStreamTrack.readyState !== "ended"
|
||||
) {
|
||||
// Restart the track, which will cause Livekit to do another
|
||||
// getUserMedia() call with deviceId: default to get the *new* default device.
|
||||
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
|
||||
// the deviceId hasn't changed (was & still is default).
|
||||
lkRoom.localParticipant
|
||||
.getTrackPublication(Track.Source.Microphone)
|
||||
?.audioTrack?.restartTrack()
|
||||
.catch((e) => {
|
||||
this.logger?.error(`Failed to restart audio device track`, e);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Observe changes in the selected media devices and update the LiveKit room accordingly.
|
||||
private observeMediaDevices(
|
||||
scope: ObservableScope,
|
||||
devices: MediaDevices,
|
||||
controlledAudioDevices: boolean,
|
||||
): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
const syncDevice = (
|
||||
kind: MediaDeviceKind,
|
||||
selected$: Observable<SelectedDevice | undefined>,
|
||||
): Subscription =>
|
||||
selected$.pipe(scope.bind()).subscribe((device) => {
|
||||
if (lkRoom.state != LivekitConnectionState.Connected) return;
|
||||
// if (this.connectionState$.value !== ConnectionState.Connected) return;
|
||||
this.logger?.info(
|
||||
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
|
||||
lkRoom.getActiveDevice(kind),
|
||||
" !== ",
|
||||
device?.id,
|
||||
);
|
||||
if (
|
||||
device !== undefined &&
|
||||
lkRoom.getActiveDevice(kind) !== device.id
|
||||
) {
|
||||
lkRoom
|
||||
.switchActiveDevice(kind, device.id)
|
||||
.catch((e) =>
|
||||
this.logger?.error(
|
||||
`Failed to sync ${kind} device with LiveKit`,
|
||||
e,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
syncDevice("audioinput", devices.audioInput.selected$);
|
||||
if (!controlledAudioDevices)
|
||||
syncDevice("audiooutput", devices.audioOutput.selected$);
|
||||
syncDevice("videoinput", devices.videoInput.selected$);
|
||||
}
|
||||
|
||||
/**
|
||||
* Observe changes in the mute states and update the LiveKit room accordingly.
|
||||
* @param scope
|
||||
* @private
|
||||
*/
|
||||
private observeMuteStates(scope: ObservableScope): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
this.muteStates.audio.setHandler(async (desired) => {
|
||||
try {
|
||||
await lkRoom.localParticipant.setMicrophoneEnabled(desired);
|
||||
} catch (e) {
|
||||
this.logger?.error(
|
||||
"Failed to update LiveKit audio input mute state",
|
||||
e,
|
||||
);
|
||||
}
|
||||
return lkRoom.localParticipant.isMicrophoneEnabled;
|
||||
});
|
||||
this.muteStates.video.setHandler(async (desired) => {
|
||||
try {
|
||||
await lkRoom.localParticipant.setCameraEnabled(desired);
|
||||
} catch (e) {
|
||||
this.logger?.error(
|
||||
"Failed to update LiveKit video input mute state",
|
||||
e,
|
||||
);
|
||||
}
|
||||
return lkRoom.localParticipant.isCameraEnabled;
|
||||
});
|
||||
}
|
||||
|
||||
private observeTrackProcessors(
|
||||
scope: ObservableScope,
|
||||
room: LivekitRoom,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
): void {
|
||||
const track$ = scope.behavior(
|
||||
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
|
||||
map((trackRef) => {
|
||||
const track = trackRef?.publication?.track;
|
||||
return track instanceof LocalVideoTrack ? track : null;
|
||||
}),
|
||||
),
|
||||
);
|
||||
trackProcessorSync(track$, trackerProcessorState$);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user