Merge branch 'livekit' into valere/initial_mute_states

This commit is contained in:
Valere
2026-01-12 19:13:40 +01:00
52 changed files with 1666 additions and 764 deletions

View File

@@ -78,11 +78,14 @@ vi.mock("../e2ee/matrixKeyProvider");
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
vi.mock("../rtcSessionHelpers", async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}));
vi.mock(
"../state/CallViewModel/localMember/localTransport",
async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}),
);
const yesNo = {
y: true,
@@ -232,7 +235,7 @@ const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
describe.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Compatibility],
[MatrixRTCMode.Matrix_2_0],
])("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
@@ -1255,11 +1258,6 @@ describe.each([
y: () => {
rtcSession.membershipStatus = Status.Connected;
},
n: () => {
// NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rtcSession.membershipStatus = "Reconnecting" as any;
},
});
schedule(probablyLeftMarbles, {
y: () => {

View File

@@ -41,10 +41,13 @@ import {
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
MembershipManagerEvent,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { v4 as uuidv4 } from "uuid";
import {
LocalUserMediaViewModel,
@@ -98,7 +101,7 @@ import {
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "../layout-types.ts";
import { ElementCallError } from "../../utils/errors.ts";
import { ElementCallError, UnknownCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
@@ -106,13 +109,19 @@ import {
enterRTCSession,
TransportState,
} from "./localMember/LocalMember.ts";
import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
import {
createLocalTransport$,
JwtEndpointVersion,
} from "./localMember/LocalTransport.ts";
import {
createMemberships$,
membershipsAndTransports$,
} from "../SessionBehaviors.ts";
import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import {
type ConnectionManagerData,
createConnectionManager$,
} from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
type TaggedParticipant,
@@ -261,6 +270,7 @@ export interface CallViewModel {
* multiple devices.
*/
participantCount$: Behavior<number>;
allConnections$: Behavior<ConnectionManagerData>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
@@ -381,8 +391,11 @@ export function createCallViewModel$(
trackProcessorState$: Behavior<ProcessorState>,
): CallViewModel {
const client = matrixRoom.client;
const userId = client.getUserId()!;
const deviceId = client.getDeviceId()!;
const userId = client.getUserId();
const deviceId = client.getDeviceId();
if (!(userId && deviceId))
throw new UnknownCallError(new Error("userId and deviceId are required"));
const livekitKeyProvider = getE2eeKeyProvider(
options.encryptionSystem,
matrixRTCSession,
@@ -415,11 +428,37 @@ export function createCallViewModel$(
memberships$,
);
const ownMembershipIdentity: CallMembershipIdentityParts = {
userId,
deviceId,
// This will only be consumed by the sticky membership manager. So it has no impact on legacy calls.
memberId: uuidv4(),
};
const localTransport$ = createLocalTransport$({
scope: scope,
memberships$: memberships$,
ownMembershipIdentity,
client,
delayId$: scope.behavior(
(
fromEvent(
matrixRTCSession,
MembershipManagerEvent.DelayIdChanged,
) as Observable<string | undefined>
).pipe(map((v) => v ?? null)),
matrixRTCSession.delayId ?? null,
),
roomId: matrixRoom.roomId,
forceJwtEndpoint$: scope.behavior(
matrixRTCMode$.pipe(
map((v) =>
v === MatrixRTCMode.Matrix_2_0
? JwtEndpointVersion.Matrix_2_0
: JwtEndpointVersion.Legacy,
),
),
),
useOldestMember$: scope.behavior(
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
@@ -439,30 +478,20 @@ export function createCallViewModel$(
const connectionManager = createConnectionManager$({
scope: scope,
connectionFactory: connectionFactory,
inputTransports$: scope.behavior(
combineLatest(
[
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"dont pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
membershipsAndTransports.transports$,
],
(localTransport, transports) => {
const localTransportAsArray = localTransport ? [localTransport] : [];
return transports.mapInner((transports) => [
...localTransportAsArray,
...transports,
]);
},
localTransport$: scope.behavior(
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"could not pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
),
logger,
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -493,6 +522,7 @@ export function createCallViewModel$(
joinMatrixRTC: (transport: LivekitTransport) => {
return enterRTCSession(
matrixRTCSession,
ownMembershipIdentity,
transport,
connectOptions$.value,
);
@@ -604,15 +634,14 @@ export function createCallViewModel$(
),
);
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value;
switchMap((members) => {
const a$ = combineLatest(
members.map((member) =>
members.value.map((member) =>
combineLatest([member.connection$, member.participant.value$]).pipe(
map(([connection, participant]) => {
// do not render audio for local participant
@@ -685,29 +714,29 @@ export function createCallViewModel$(
generateItems(
function* ([
localMatrixLivekitMember,
{ value: matrixLivekitMembers },
matrixLivekitMembers,
duplicateTiles,
]) {
let localParticipantId: string | undefined = undefined;
let localUserMediaId: string | undefined = undefined;
// add local member if available
if (localMatrixLivekitMember) {
const { userId, participant, connection$, membership$ } =
localMatrixLivekitMember;
localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
// const participantId = membership$.value.membershipID;
if (localParticipantId) {
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [
dup,
localParticipantId,
userId,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
],
data: undefined,
};
}
localUserMediaId = `${userId}:${membership$.value.deviceId}`;
const rtcBackendIdentity = membership$.value.rtcBackendIdentity;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [
dup,
localUserMediaId,
userId,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
rtcBackendIdentity,
],
data: undefined,
};
}
}
// add remote members that are available
@@ -716,13 +745,22 @@ export function createCallViewModel$(
participant,
connection$,
membership$,
} of matrixLivekitMembers) {
const participantId = `${userId}:${membership$.value.deviceId}`;
if (participantId === localParticipantId) continue;
// const participantId = membership$.value?.identity;
} of matrixLivekitMembers.value) {
const userMediaId = `${userId}:${membership$.value.deviceId}`;
const rtcBackendIdentity = membership$.value.rtcBackendIdentity;
// skip local user as we added them manually before
if (userMediaId === localUserMediaId) continue;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, participantId, userId, participant, connection$],
keys: [
dup,
userMediaId,
userId,
participant,
connection$,
rtcBackendIdentity,
],
data: undefined,
};
}
@@ -732,10 +770,11 @@ export function createCallViewModel$(
scope,
_data$,
dup,
participantId,
userMediaId,
userId,
participant,
connection$,
rtcBackendIdentity,
) => {
const livekitRoom$ = scope.behavior(
connection$.pipe(map((c) => c?.livekitRoom)),
@@ -751,8 +790,9 @@ export function createCallViewModel$(
return new UserMedia(
scope,
`${participantId}:${dup}`,
`${userMediaId}:${dup}`,
userId,
rtcBackendIdentity,
participant,
options.encryptionSystem,
livekitRoom$,
@@ -761,8 +801,8 @@ export function createCallViewModel$(
localMembership.reconnecting$,
displayName$,
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
reactions$.pipe(map((v) => v[participantId] ?? undefined)),
handsRaised$.pipe(map((v) => v[userMediaId]?.time ?? null)),
reactions$.pipe(map((v) => v[userMediaId] ?? undefined)),
);
},
),
@@ -1503,6 +1543,7 @@ export function createCallViewModel$(
),
null,
),
allConnections$,
participantCount$: participantCount$,
handsRaised$: handsRaised$,
reactions$: reactions$,

View File

@@ -24,6 +24,7 @@ import {
mockLivekitRoom,
mockMuteStates,
withTestScheduler,
ownMemberMock,
} from "../../../utils/test";
import {
TransportState,
@@ -38,6 +39,7 @@ import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
import { type LocalTransportWithSFUConfig } from "./LocalTransport";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
@@ -103,11 +105,12 @@ describe("LocalMembership", () => {
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -119,7 +122,12 @@ describe("LocalMembership", () => {
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
expect(mockedSession.joinRTCSession).toHaveBeenLastCalledWith(
{
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
userId: "@alice:example.org",
},
[
{
livekit_alias: "roomId",
@@ -161,11 +169,12 @@ describe("LocalMembership", () => {
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -204,10 +213,11 @@ describe("LocalMembership", () => {
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const localTransport$ = scope.behavior<null | LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
const localTransport$ =
scope.behavior<null | LocalTransportWithSFUConfig>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
// we do not need any connection data since we want to fail before reaching that.
const mockConnectionManager = {
@@ -235,11 +245,23 @@ describe("LocalMembership", () => {
});
const aTransport = {
livekit_service_url: "a",
} as LivekitTransport;
transport: {
livekit_service_url: "a",
} as LivekitTransport,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const bTransport = {
livekit_service_url: "b",
} as LivekitTransport;
transport: {
livekit_service_url: "b",
} as LivekitTransport,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
@@ -249,7 +271,7 @@ describe("LocalMembership", () => {
} as unknown as LocalParticipant,
}),
state$: constant(ConnectionState.LivekitConnected),
transport: aTransport,
transport: aTransport.transport,
} as unknown as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
@@ -258,11 +280,11 @@ describe("LocalMembership", () => {
} as unknown as Connection;
const connectionTransportBConnected = {
state$: constant(ConnectionState.LivekitConnected),
transport: bTransport,
transport: bTransport.transport,
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
it("recreates publisher if new connection is used, always unpublish and end tracks", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
@@ -310,8 +332,12 @@ describe("LocalMembership", () => {
expect(publishers[1].stopTracks).not.toHaveBeenCalled();
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopPublishing).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport);
expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport);
expect(publisherFactory.mock.calls[0][0].transport).toBe(
aTransport.transport,
);
expect(publisherFactory.mock.calls[1][0].transport).toBe(
bTransport.transport,
);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
@@ -383,7 +409,8 @@ describe("LocalMembership", () => {
const scope = new ObservableScope();
const connectionManagerData = new ConnectionManagerData();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const localTransport$ =
new BehaviorSubject<null | LocalTransportWithSFUConfig>(null);
const connectionManagerData$ = new BehaviorSubject(
new Epoch(connectionManagerData),
);
@@ -460,7 +487,7 @@ describe("LocalMembership", () => {
});
(
connectionManagerData2.getConnectionForTransport(aTransport)!
connectionManagerData2.getConnectionForTransport(aTransport.transport)!
.state$ as BehaviorSubject<ConnectionState>
).next(ConnectionState.LivekitConnected);
expect(localMembership.localMemberState$.value).toStrictEqual({

View File

@@ -36,6 +36,7 @@ import {
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
@@ -60,6 +61,7 @@ import {
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts";
export enum TransportState {
/** Not even a transport is available to the LocalMembership */
@@ -125,7 +127,7 @@ interface Props {
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => void;
homeserverConnected: HomeserverConnected;
localTransport$: Behavior<LivekitTransport | null>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
@@ -233,7 +235,9 @@ export const createLocalMembership$ = ({
return null;
}
return connectionData.getConnectionForTransport(localTransport);
return connectionData.getConnectionForTransport(
localTransport.transport,
);
}),
tap((connection) => {
logger.info(
@@ -532,7 +536,7 @@ export const createLocalMembership$ = ({
if (!shouldConnect) return;
try {
joinMatrixRTC(transport);
joinMatrixRTC(transport.transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
@@ -551,7 +555,12 @@ export const createLocalMembership$ = ({
);
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
localConnection$.pipe(
map((c) => c?.livekitRoom?.localParticipant ?? null),
tap((p) => {
logger.debug("participant$ updated:", p?.identity);
}),
),
);
// Pause upstream of all local media tracks when we're disconnected from
@@ -686,18 +695,19 @@ interface EnterRTCSessionOptions {
* - Handles retries (fails only after several attempts)
*
* @param rtcSession - The MatrixRTCSession to join.
* @param ownMembershipIdentity - Options for entering the RTC session.
* @param transport - The LivekitTransport to use for this session.
* @param options - Options for entering the RTC session.
* @param options.encryptMedia - Whether to encrypt media.
* @param options.matrixRTCMode - The Matrix RTC mode to use.
* @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use.
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export function enterRTCSession(
rtcSession: MatrixRTCSession,
ownMembershipIdentity: CallMembershipIdentityParts,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
options: EnterRTCSessionOptions,
): void {
const { encryptMedia, matrixRTCMode } = options;
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -709,10 +719,13 @@ export function enterRTCSession(
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
const multiSFU =
matrixRTCMode === MatrixRTCMode.Compatibility ||
matrixRTCMode === MatrixRTCMode.Matrix_2_0;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
rtcSession.joinRTCSession(
ownMembershipIdentity,
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{

View File

@@ -18,8 +18,8 @@ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import { mockConfig, flushPromises } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport";
import { constant } from "../../Behavior";
import { Epoch, ObservableScope } from "../../ObservableScope";
import {
@@ -39,11 +39,35 @@ describe("LocalTransport", () => {
};
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
it("throws if config is missing", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
baseUrl: "example.org",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
// Provide a valid config so makeTransportInternal resolves a transport
const scope = new ObservableScope();
@@ -65,6 +89,7 @@ describe("LocalTransport", () => {
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
@@ -72,6 +97,9 @@ describe("LocalTransport", () => {
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
localTransport$.subscribe(
(o) => observations.push(o),
@@ -86,6 +114,60 @@ describe("LocalTransport", () => {
expect(() => localTransport$.value).toThrow(expectedError);
});
it("emits preferred transport after OpenID resolves", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.({
url: "https://lk.example.org",
jwt: "jwt",
livekitAlias: "!room:example.org",
livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId,
});
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
transport: {
livekit_alias: "!room:example.org",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "jwt",
livekitAlias: "!room:example.org",
livekitIdentity: "@alice:example.org:DEVICE",
url: "https://lk.example.org",
},
});
});
it("updates local transport when oldest member changes", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
@@ -109,7 +191,11 @@ describe("LocalTransport", () => {
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.(openIdResponse);
@@ -117,9 +203,17 @@ describe("LocalTransport", () => {
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
@@ -134,11 +228,15 @@ describe("LocalTransport", () => {
mockConfig({});
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
ownMembershipIdentity: ownMemberMock,
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://example.org",
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
@@ -165,9 +263,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via user settings", async () => {
@@ -177,9 +283,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via backend", async () => {
@@ -191,9 +305,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("fails fast if the openID request fails for backend config", async () => {
@@ -222,9 +344,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
expect(fetchMock.done()).toEqual(true);
});
@@ -248,11 +378,15 @@ describe("LocalTransport", () => {
it("throws if no options are available", async () => {
const localTransport$ = createLocalTransport$({
scope,
ownMembershipIdentity: ownMemberMock,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
// These won't be called in this error path but satisfy the type

View File

@@ -23,6 +23,7 @@ import {
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
@@ -30,9 +31,11 @@ import { Config } from "../../../config/Config.ts";
import {
FailToGetOpenIdToken,
MatrixRTCTransportMissingError,
NoMatrix2AuthorizationService,
} from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type SFUConfig,
type OpenIDClientParts,
} from "../../../livekit/openIDSFU.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
@@ -47,11 +50,32 @@ const logger = rootLogger.getChild("[LocalTransport]");
*/
interface Props {
scope: ObservableScope;
ownMembershipIdentity: CallMembershipIdentityParts;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts;
roomId: string;
useOldestMember$: Behavior<boolean>;
forceJwtEndpoint$: Behavior<JwtEndpointVersion>;
delayId$: Behavior<string | null>;
}
export enum JwtEndpointVersion {
Legacy = "legacy",
Matrix_2_0 = "matrix_2_0",
}
export interface LocalTransportWithSFUConfig {
transport: LivekitTransport;
sfuConfig: SFUConfig;
}
export function isLocalTransportWithSFUConfig(
obj: LivekitTransport | LocalTransportWithSFUConfig,
): obj is LocalTransportWithSFUConfig {
return "transport" in obj && "sfuConfig" in obj;
}
/**
@@ -61,26 +85,53 @@ interface Props {
* @prop useOldestMember Whether to use the same transport as the oldest member.
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
*
* @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint.
* This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity.
* (which is expected for non sticky event based rtc member events)
* @returns The local transport. It will be created using the correct sfu endpoint based on the useOldJwtEndpoint$ value.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
export const createLocalTransport$ = ({
scope,
memberships$,
ownMembershipIdentity,
client,
roomId,
useOldestMember$,
}: Props): Behavior<LivekitTransport | null> => {
forceJwtEndpoint$,
delayId$,
}: Props): Behavior<LocalTransportWithSFUConfig | null> => {
/**
* The transport over which we should be actively publishing our media.
* undefined when not joined.
*/
const oldestMemberTransport$ = scope.behavior(
memberships$.pipe(
map(
(memberships) =>
memberships.value[0]?.getTransport(memberships.value[0]) ?? null,
),
combineLatest([memberships$]).pipe(
map(([memberships]) => {
const oldestMember = memberships.value[0];
const transport = oldestMember?.getTransport(memberships.value[0]);
if (!transport) return null;
return transport;
}),
first((t) => t != null && isLivekitTransport(t)),
switchMap((transport) => {
// Get the open jwt token to connect to the sfu
const computeLocalTransportWithSFUConfig =
async (): Promise<LocalTransportWithSFUConfig> => {
return {
transport,
sfuConfig: await getSFUConfigWithOpenID(
client,
ownMembershipIdentity,
transport.livekit_service_url,
roomId,
{ forceJwtEndpoint: JwtEndpointVersion.Legacy },
logger,
),
};
};
return from(computeLocalTransportWithSFUConfig());
}),
),
null,
);
@@ -91,9 +142,30 @@ export const createLocalTransport$ = ({
*
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(
switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))),
const preferredTransport$ = scope.behavior(
// preferredTransport$ (used for multi sfu) needs to know if we are using the old or new
// jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity
// differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`)
// When using sticky events (we need to use the new endpoint).
combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe(
switchMap(([customUrl, delayId, forceEndpoint]) => {
logger.info(
"Creating preferred transport based on: ",
customUrl,
delayId,
forceEndpoint,
);
return from(
makeTransport(
client,
ownMembershipIdentity,
roomId,
customUrl,
forceEndpoint,
delayId ?? undefined,
),
);
}),
),
null,
);
@@ -112,7 +184,9 @@ export const createLocalTransport$ = ({
? (oldestMemberTransport ?? preferredTransport)
: preferredTransport,
),
distinctUntilChanged(areLivekitTransportsEqual),
distinctUntilChanged((t1, t2) =>
areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null),
),
),
);
};
@@ -124,25 +198,63 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
* validating auth against the service to ensure it's correct.
* Prefers in order:
*
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
*
* @param client The authenticated Matrix client for the current user
* @param membership The membership identity of the user.
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @param forceJwtEndpoint Whether to force a specific JWT endpoint
* - `Legacy` / `Matrix_2_0`
* - `get_token` / `sfu/get`
* - not hashing / hashing the backendIdentity
* @param delayId the delay id passed to the jwt service.
*
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
membership: CallMembershipIdentityParts,
roomId: string,
urlFromDevSettings: string | null,
): Promise<LivekitTransport> {
forceJwtEndpoint: JwtEndpointVersion,
delayId?: string,
): Promise<LocalTransportWithSFUConfig> {
logger.trace("Searching for a preferred transport");
async function doOpenIdAndJWTFromUrl(
url: string,
): Promise<LocalTransportWithSFUConfig> {
const sfuConfig = await getSFUConfigWithOpenID(
client,
membership,
url,
roomId,
{
forceJwtEndpoint: forceJwtEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
},
logger,
);
return {
transport: {
type: "livekit",
livekit_service_url: url,
livekit_alias: sfuConfig.livekitAlias,
},
sfuConfig,
};
}
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
@@ -153,39 +265,29 @@ async function makeTransport(
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
const config = await getSFUConfigWithOpenID(
client,
urlFromDevSettings,
roomId,
);
return {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: config.livekitAlias,
};
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
return await doOpenIdAndJWTFromUrl(urlFromDevSettings);
}
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LivekitTransport | null> {
): Promise<LocalTransportWithSFUConfig | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
return await doOpenIdAndJWTFromUrl(
potentialTransport.livekit_service_url,
roomId,
);
return {
...potentialTransport,
livekit_alias: livekitAlias,
};
} catch (ex) {
// Explictly throw these
if (ex instanceof FailToGetOpenIdToken) {
// Explictly throw these
throw ex;
}
if (ex instanceof NoMatrix2AuthorizationService) {
throw ex;
}
logger.debug(
@@ -245,18 +347,9 @@ async function makeTransport(
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
urlFromConf,
roomId,
);
const selectedTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using config SFU", selectedTransport);
return selectedTransport;
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using config SFU", urlFromConf);
return await doOpenIdAndJWTFromUrl(urlFromConf);
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
@@ -265,5 +358,6 @@ async function makeTransport(
}
}
// If we do not have returned a transport by now we throw an error
throw new MatrixRTCTransportMissingError(domain ?? "");
}

View File

@@ -26,8 +26,8 @@ import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
@@ -40,7 +40,7 @@ import {
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant } from "../../../utils/test.ts";
import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -155,6 +156,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
@@ -170,6 +172,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -220,6 +223,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -259,7 +263,7 @@ describe("Start connection states", () => {
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception",
"SFU Config fetch failed with status code 500",
);
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
@@ -277,6 +281,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};

View File

@@ -18,6 +18,7 @@ import {
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import {
getSFUConfigWithOpenID,
@@ -32,8 +33,21 @@ import {
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts";
export interface ConnectionOpts {
/**
* For the local transport we already do know the jwt token and url. We can reuse it.
* On top the local transport will send additional data to the jwt server to use delayed event delegation.
*/
existingSFUConfig?: SFUConfig;
/**
* For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig
* and hence we need to let the connection do the jwt token fetching.
*/
forceJwtEndpoint?: JwtEndpointVersion;
/** The identity parts to use on this connection */
ownMembershipIdentity: CallMembershipIdentityParts;
/** The media transport to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
@@ -129,8 +143,10 @@ export class Connection {
try {
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// It would probably be better to forward this here.
const { url, jwt } = await this.getSFUConfigWithOpenID();
// only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined)
const { url, jwt } =
this.existingSFUConfig ??
(await this.getSFUConfigForRemoteConnection());
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
@@ -186,11 +202,17 @@ export class Connection {
}
}
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
protected async getSFUConfigForRemoteConnection(): Promise<SFUConfig> {
// This will only be called for sfu's where we do not publish ourselves.
// For the local connection we will use the existingJwtTokenData
return await getSFUConfigWithOpenID(
this.client,
this.ownMembershipIdentity,
this.transport.livekit_service_url,
this.transport.livekit_alias,
// dont pass any custom opts for the subscribe only connections
{},
this.logger,
);
}
@@ -212,7 +234,8 @@ export class Connection {
private readonly client: OpenIDClientParts;
private readonly logger: Logger;
private readonly ownMembershipIdentity: CallMembershipIdentityParts;
private readonly existingSFUConfig?: SFUConfig;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
@@ -221,6 +244,8 @@ export class Connection {
* @param logger - The logger to use.
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.ownMembershipIdentity = opts.ownMembershipIdentity;
this.existingSFUConfig = opts.existingSFUConfig;
this.logger = logger.getChild("[Connection]");
this.logger.info(
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,

View File

@@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Room as LivekitRoom,
type RoomOptions,
@@ -16,10 +15,15 @@ import {
import { type Logger } from "matrix-js-sdk/lib/logger";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import type {
OpenIDClientParts,
SFUConfig,
} from "../../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
@@ -28,9 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransport,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection;
}
@@ -78,17 +84,30 @@ export class ECConnectionFactory implements ConnectionFactory {
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
/**
*
* @param scope The observable scope (used for clean-up)
* @param transport The transport to use for this connection.
* @param ownMembershipIdentity required to connect (using the jwt service) with the SFU.
* @param logger The logger instance to use for this connection.
* @param sfuConfig optional config in case we already have a token for this connection.
* @returns
*/
public createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransport,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection {
return new Connection(
{
existingSFUConfig: sfuConfig,
transport,
client: this.client,
scope: scope,
livekitRoomFactory: this.livekitRoomFactory,
ownMembershipIdentity,
},
logger,
);

View File

@@ -18,9 +18,9 @@ import {
} from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { withTestScheduler } from "../../../utils/test.ts";
import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type Behavior } from "../../Behavior.ts";
import { constant, type Behavior } from "../../Behavior.ts";
// Some test constants
@@ -49,7 +49,7 @@ beforeEach(() => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransport) => {
const mockConnection = {
transport,
remoteParticipants$: new BehaviorSubject([]),
@@ -76,10 +76,12 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -115,7 +117,8 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abcdef", {
localTransport$: constant(null),
remoteTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
c: new Epoch([TRANSPORT_1], 2),
@@ -124,6 +127,7 @@ describe("connections$ stream", () => {
f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -160,12 +164,14 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abc", {
localTransport$: constant(null),
remoteTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
c: new Epoch([TRANSPORT_1], 2),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -223,7 +229,7 @@ describe("connectionManagerData$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransport) => {
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
@@ -275,10 +281,12 @@ describe("connectionManagerData$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionManagerData$).toBe("abcd", {

View File

@@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { combineLatest, map, of, switchMap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type RemoteParticipant } from "livekit-client";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
@@ -17,6 +18,11 @@ import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateItemsWithEpoch } from "../../../utils/observable.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import {
isLocalTransportWithSFUConfig,
type LocalTransportWithSFUConfig,
} from "../localMember/LocalTransport.ts";
import { type SFUConfig } from "../../../livekit/openIDSFU.ts";
export class ConnectionManagerData {
private readonly store: Map<
@@ -65,8 +71,11 @@ export class ConnectionManagerData {
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
remoteTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
ownMembershipIdentity: CallMembershipIdentityParts;
}
// TODO - write test for scopes (do we really need to bind scope)
@@ -79,8 +88,12 @@ export interface IConnectionManager {
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
* @param props.logger - The logger to use
* @param props.localTransport$ - The local transport to use. (deduplicated with remoteTransports$)
* @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$)
* @param props.ownMembershipIdentity - The own membership identity to use.
* @param props.logger - The logger to use.
*
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
@@ -93,8 +106,10 @@ export interface IConnectionManager {
export function createConnectionManager$({
scope,
connectionFactory,
inputTransports$,
localTransport$,
remoteTransports$,
logger: parentLogger,
ownMembershipIdentity,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
@@ -107,12 +122,33 @@ export function createConnectionManager$({
* It is build based on the list of subscribed transports (`transportsSubscriptions$`).
* externally this is modified via `registerTransports()`.
*/
const transports$ = scope.behavior(
inputTransports$.pipe(
map((transports) => transports.mapInner(removeDuplicateTransports)),
tap(({ value: transports }) => {
logger.trace(
`Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`,
const localAndRemoteTransports$: Behavior<
Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]>
> = scope.behavior(
combineLatest([remoteTransports$, localTransport$]).pipe(
// Combine local and remote transports into one transport array
// and set the forceOldJwtEndpoint property on the local transport
map(([remoteTransports, localTransport]) => {
let localTransportAsArray: LocalTransportWithSFUConfig[] = [];
if (localTransport) {
localTransportAsArray = [localTransport];
}
const dedupedRemote = removeDuplicateTransports(remoteTransports.value);
const remoteWithoutLocal = dedupedRemote.filter(
(transport) =>
!localTransportAsArray.find((l) =>
areLivekitTransportsEqual(l.transport, transport),
),
);
logger.debug(
"remoteWithoutLocal",
remoteWithoutLocal,
"localTransportAsArray",
localTransportAsArray,
);
return new Epoch(
[...localTransportAsArray, ...remoteWithoutLocal],
remoteTransports.epoch,
);
}),
),
@@ -122,25 +158,51 @@ export function createConnectionManager$({
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
transports$.pipe(
localAndRemoteTransports$.pipe(
generateItemsWithEpoch(
function* (transports) {
for (const transport of transports)
yield {
keys: [transport.livekit_service_url, transport.livekit_alias],
data: undefined,
};
for (const transportWithOrWithoutSfuConfig of transports) {
if (
isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig)
) {
// This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field
const { transport, sfuConfig } = transportWithOrWithoutSfuConfig;
yield {
keys: [
transport.livekit_service_url,
transport.livekit_alias,
sfuConfig,
],
data: undefined,
};
} else {
const transport = transportWithOrWithoutSfuConfig;
yield {
keys: [
transport.livekit_service_url,
transport.livekit_alias,
undefined as undefined | SFUConfig,
],
data: undefined,
};
}
}
},
(scope, _data$, serviceUrl, alias) => {
logger.debug(`Creating connection to ${serviceUrl} (${alias})`);
(scope, _data$, serviceUrl, alias, sfuConfig) => {
logger.debug(
`Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`,
);
const connection = connectionFactory.createConnection(
scope,
{
type: "livekit",
livekit_service_url: serviceUrl,
livekit_alias: alias,
},
scope,
ownMembershipIdentity,
logger,
sfuConfig,
);
// Start the connection immediately
// Use connection state to track connection progress
@@ -190,18 +252,18 @@ export function createConnectionManager$({
);
}),
),
new Epoch(new ConnectionManagerData()),
new Epoch(new ConnectionManagerData(), -1),
);
return { connectionManagerData$ };
}
function removeDuplicateTransports(
transports: LivekitTransport[],
): LivekitTransport[] {
function removeDuplicateTransports<T extends LivekitTransport>(
transports: T[],
): T[] {
return transports.reduce((acc, transport) => {
if (!acc.some((t) => areLivekitTransportsEqual(t, transport)))
acc.push(transport);
return acc;
}, [] as LivekitTransport[]);
}, [] as T[]);
}

View File

@@ -15,7 +15,11 @@ import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts";
import {
exampleTransport,
mockMediaDevices,
ownMemberMock,
} from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
@@ -72,7 +76,12 @@ describe("ECConnectionFactory - Audio inputs options", () => {
echo,
noise,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
@@ -113,7 +122,12 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
false,
false,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(

View File

@@ -10,8 +10,7 @@ import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
import { combineLatest, map, type Observable } from "rxjs";
import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
@@ -26,14 +25,18 @@ import {
} from "../../ObservableScope.ts";
import { ConnectionManagerData } from "./ConnectionManager.ts";
import {
mockCallMembership,
flushPromises,
mockRtcMembership,
mockRemoteParticipant,
withTestScheduler,
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
import { constant } from "../../Behavior.ts";
let testScope: ObservableScope;
const fallbackMemberId = (userId: string, deviceId: string): string =>
`${userId}:${deviceId}`;
const transportA: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
@@ -46,16 +49,12 @@ const transportB: LivekitTransport = {
livekit_alias: "!alias:sample.com",
};
const bobMembership = mockCallMembership(
"@bob:example.org",
"DEV000",
transportA,
);
const carlMembership = mockCallMembership(
"@carl:sample.com",
"DEV111",
transportB,
);
const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [transportA],
});
const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", {
fociPreferred: [transportB],
});
beforeEach(() => {
testScope = new ObservableScope();
@@ -76,52 +75,41 @@ function epochMeWith$<T, U>(
);
}
test("should signal participant not yet connected to livekit", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("should signal participant not yet connected to livekit", async () => {
const mockedMemberships$ = new BehaviorSubject([bobMembership]);
const mockConnectionManagerData$ = new BehaviorSubject(
new ConnectionManagerData(),
);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: new ConnectionManagerData(),
}),
);
const connectionManagerData$ = epochMeWith$(
memberships$,
mockConnectionManagerData$,
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(null);
return true;
},
);
});
// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable.
function fromMemberships$(m$: Observable<CallMembership[]>): {
function createEpochedMemberships$(m$: Observable<CallMembership[]>): {
memberships$: Observable<Epoch<CallMembership[]>>;
membershipsWithTransport$: Observable<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
@@ -146,32 +134,115 @@ function fromMemberships$(m$: Observable<CallMembership[]>): {
};
}
test("should signal participant on a connection that is publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobParticipantId = getParticipantId(
test("should signal participant on a connection that is publishing", async () => {
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
});
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
test("should signal participant on a connection that is not publishing", async () => {
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", async () => {
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(constant([bobMembership, carlMembership]));
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
const connectionA = {
transport: transportA,
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
constant(connectionWithPublisher),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -181,213 +252,73 @@ test("should signal participant on a connection that is publishing", () => {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
await flushPromises();
expect(matrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
},
);
});
});
test("should signal participant on a connection that is not publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("bob is publishing in the wrong connection", async () => {
const mockedMemberships$ = new BehaviorSubject([
bobMembership,
carlMembership,
]);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
} as unknown as Connection;
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
}),
},
);
});
});
test("bob is publishing in the wrong connection", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});
return true;
}),
},
);
});
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionsWithPublisher$ = new BehaviorSubject(
connectionWithPublisher,
);
const connectionManagerData$ = epochMeWith$(
memberships$,
connectionsWithPublisher$,
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toBe(null);
return true;
},
);
});

View File

@@ -84,7 +84,6 @@ export function createMatrixLivekitMembers$({
/**
* Stream of all the call members and their associated livekit data (if available).
*/
return scope.behavior(
combineLatest([
membershipsWithTransport$,
@@ -93,47 +92,39 @@ export function createMatrixLivekitMembers$({
filter((values) =>
values.every((value) => value.epoch === values[0].epoch),
),
map(
([
{ value: membershipsWithTransports, epoch },
{ value: managerData },
]) =>
new Epoch([membershipsWithTransports, managerData] as const, epoch),
),
map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)),
generateItemsWithEpoch(
// Generator function.
// creates an array of `{key, data}[]`
// Each change in the keys (new key, missing key) will result in a call to the factory function.
function* ([membershipsWithTransports, managerData]) {
for (const { membership, transport } of membershipsWithTransports) {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
function* ([membershipsWithTransport, managerData]) {
for (const { membership, transport } of membershipsWithTransport) {
const participants = transport
? managerData.getParticipantsForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;
participants.find(
(p) => p.identity == membership.rtcBackendIdentity,
) ?? null;
const connection = transport
? managerData.getConnectionForTransport(transport)
: null;
yield {
keys: [participantId, membership.userId],
keys: [membership.userId, membership.deviceId],
data: { membership, participant, connection },
};
}
},
// Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
(scope, data$, participantId, userId) => {
(scope, data$, userId, deviceId) => {
logger.debug(
`Generating member for participantId: ${participantId}, userId: ${userId}`,
`Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`,
);
const { participant$, ...rest } = scope.splitBehavior(data$);
// will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
participantId,
userId,
participant: { type: "remote" as const, value$: participant$ },
...rest,
@@ -141,15 +132,16 @@ export function createMatrixLivekitMembers$({
},
),
),
new Epoch([], -1),
);
}
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
// TODO add this to the JS-SDK
export function areLivekitTransportsEqual(
t1: LivekitTransport | null,
t2: LivekitTransport | null,
export function areLivekitTransportsEqual<T extends LivekitTransport>(
t1: T | null,
t2: T | null,
): boolean {
if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url;
// In case we have different lk rooms in the same SFU (depends on the livekit authorization service)

View File

@@ -18,7 +18,7 @@ import { it } from "vitest";
import { ObservableScope } from "../../ObservableScope.ts";
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
import {
mockCallMembership,
mockRtcMembership,
mockMatrixRoomMember,
withTestScheduler,
} from "../../../utils/test.ts";
@@ -111,7 +111,7 @@ describe("MatrixMemberMetadata", () => {
rawDisplayName: "it's a me",
});
const memberships$ = behavior("a", {
a: [mockCallMembership("@local:example.com", "DEVICE1")],
a: [mockRtcMembership("@local:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
@@ -149,8 +149,8 @@ describe("MatrixMemberMetadata", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@alice:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@alice:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -179,7 +179,7 @@ describe("MatrixMemberMetadata", () => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [mockCallMembership("@no-name:foo.bar", "D000")],
a: [mockRtcMembership("@no-name:foo.bar", "D000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
@@ -201,11 +201,11 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE2"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockCallMembership("@carl:example.com", "C000"),
mockCallMembership("@evil:example.com", "E000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE2"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@carl:example.com", "C000"),
mockRtcMembership("@evil:example.com", "E000"),
],
});
@@ -233,10 +233,10 @@ describe("MatrixMemberMetadata", () => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [mockCallMembership("@bob:example.com", "DEVICE1")],
a: [mockRtcMembership("@bob:example.com", "DEVICE1")],
b: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
});
@@ -262,10 +262,10 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("ab", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
b: [mockCallMembership("@bob:example.com", "DEVICE1")],
b: [mockRtcMembership("@bob:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -292,8 +292,8 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -331,16 +331,16 @@ describe("MatrixMemberMetadata", () => {
// - room join/leave
// - disambiguate
const memberships$ = behavior("ab-d", {
a: [mockCallMembership(CARL, "C000")],
a: [mockRtcMembership(CARL, "C000")],
b: [
mockCallMembership(CARL, "C000"),
mockRtcMembership(CARL, "C000"),
// bob joins
mockCallMembership(BOB, "B000"),
mockRtcMembership(BOB, "B000"),
],
// c carl gets renamed to BOB
d: [
// carl leaves
mockCallMembership(BOB, "B000"),
mockRtcMembership(BOB, "B000"),
],
});
schedule("--a-", {
@@ -379,8 +379,8 @@ describe("MatrixMemberMetadata", () => {
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockCallMembership(
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",
"BBBB",
);
@@ -397,9 +397,9 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith(bobZeroWidthSpace);
fakeMemberWith({ userId: "@carol:example.org" });
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember],
a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember],
b: [
mockCallMembership("@carol:example.org", "1111"),
mockRtcMembership("@carol:example.org", "1111"),
bobRtcMember,
bobZeroWidthSpaceRtcMember,
],
@@ -450,8 +450,8 @@ describe("MatrixMemberMetadata", () => {
it("should strip RTL characters from displayname", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockCallMembership(
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockRtcMembership(
"@dave2:example.org",
"DDDD",
);
@@ -466,9 +466,9 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith(daveRTL);
fakeMemberWith(dave);
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "DDDD")],
a: [mockRtcMembership("@carol:example.org", "DDDD")],
b: [
mockCallMembership("@carol:example.org", "DDDD"),
mockRtcMembership("@carol:example.org", "DDDD"),
daveRtcMember,
daveRTLRtcMember,
],
@@ -527,8 +527,8 @@ describe("MatrixMemberMetadata", () => {
});
const memberships$ = behavior("a", {
a: [
mockCallMembership("@local:example.com", "DEVICE1"),
mockCallMembership("@alice:example.com", "DEVICE1"),
mockRtcMembership("@local:example.com", "DEVICE1"),
mockRtcMembership("@alice:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -562,12 +562,12 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
a: [mockCallMembership("@bob:example.com", "B000")],
a: [mockRtcMembership("@bob:example.com", "B000")],
b: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
d: [mockCallMembership("@carl:example.com", "C000")],
d: [mockRtcMembership("@carl:example.com", "C000")],
});
const metadataStore = createMatrixMemberMetadata$(

View File

@@ -21,8 +21,9 @@ import {
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import {
mockCallMembership,
mockMediaDevices,
mockRtcMembership,
ownMemberMock,
withTestScheduler,
} from "../../../utils/test.ts";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
@@ -33,6 +34,7 @@ import {
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { constant } from "../../Behavior.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
@@ -99,9 +101,9 @@ afterEach(() => {
test("bob, carl, then bob joining no tracks yet", () => {
withTestScheduler(({ expectObservable, behavior, scope }) => {
const bobMembership = mockCallMembership("@bob:example.com", "BDEV000");
const carlMembership = mockCallMembership("@carl:example.com", "CDEV000");
const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000");
const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000");
const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000");
const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000");
const eMarble = "abc";
const vMarble = "abc";
@@ -121,8 +123,10 @@ test("bob, carl, then bob joining no tracks yet", () => {
const connectionManager = createConnectionManager$({
scope: testScope,
connectionFactory: ecConnectionFactory,
inputTransports$: membershipsAndTransports.transports$,
localTransport$: constant(null),
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({

View File

@@ -37,7 +37,7 @@ vi.mock("../widget", () => ({
it.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Compatibility],
[MatrixRTCMode.Matrix_2_0],
])(
"expect leave when ElementWidgetActions.HangupCall is called (%s mode)",

View File

@@ -257,6 +257,7 @@ abstract class BaseMediaViewModel {
* The Matrix user to which this media belongs.
*/
public readonly userId: string,
public readonly rtcBackendIdentity: string,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant$: Observable<
@@ -406,6 +407,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<LocalParticipant | RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -419,6 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
Track.Source.Microphone,
@@ -544,6 +547,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Behavior<LocalParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -558,6 +562,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
livekitRoom$,
@@ -671,6 +676,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -685,6 +691,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
livekitRoom$,
@@ -772,6 +779,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -785,6 +793,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
Track.Source.ScreenShareAudio,

View File

@@ -28,6 +28,7 @@ export class ScreenShare {
private readonly scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -40,6 +41,7 @@ export class ScreenShare {
this.scope,
id,
userId,
rtcBackendIdentity,
of(participant),
encryptionSystem,
livekitRoom$,

View File

@@ -75,6 +75,7 @@ export class UserMedia {
this.scope,
this.id,
this.userId,
this.rtcBackendIdentity,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
@@ -89,6 +90,7 @@ export class UserMedia {
this.scope,
this.id,
this.userId,
this.rtcBackendIdentity,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
@@ -140,6 +142,7 @@ export class UserMedia {
scope,
`${this.id}:${key}`,
this.userId,
this.rtcBackendIdentity,
p,
this.encryptionSystem,
this.livekitRoom$,
@@ -191,6 +194,7 @@ export class UserMedia {
private readonly scope: ObservableScope,
public readonly id: string,
private readonly userId: string,
private readonly rtcBackendIdentity: string,
private readonly participant: TaggedParticipant,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,