studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'livekit' into renovate/major-compound

Browse Source
This commit is contained in:
Robin
2025-05-28 18:07:30 -04:00
parent 07a4244c05 03f92ed6e4
commit 0661486676
209 changed files with 6517 additions and 1353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/@types/global.d.ts vendored
View File

@@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import "matrix-js-sdk/src/@types/global";
import { type setLogLevel as setLKLogLevel } from "livekit-client";
import type { DurationFormat as PolyfillDurationFormat } from "@formatjs/intl-durationformat";

2
src/@types/matrix-js-sdk.d.ts vendored
View File

@@ -11,7 +11,7 @@ import {
} from "../reactions";
// Extend Matrix JS SDK types via Typescript declaration merging to support unspecced event fields and types
declare module "matrix-js-sdk/src/types" {
declare module "matrix-js-sdk/lib/types" {
export interface TimelineEvents {
[ElementCallReactionEventType]: ECallReactionEventContent;
}

37
src/App.tsx
View File

@@ -9,7 +9,7 @@ import { type FC, type JSX, Suspense, useEffect, useState } from "react";
import { BrowserRouter, Route, useLocation, Routes } from "react-router-dom";
import * as Sentry from "@sentry/react";
import { TooltipProvider } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { HomePage } from "./home/HomePage";
import { LoginPage } from "./auth/LoginPage";
@@ -22,6 +22,7 @@ import { Initializer } from "./initializer";
import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
import { widget } from "./widget";
import { useTheme } from "./useTheme";
import { ProcessorProvider } from "./livekit/TrackProcessorContext";
const SentryRoute = Sentry.withSentryReactRouterV7Routing(Route);
@@ -72,22 +73,24 @@ export const App: FC = () => {
<Suspense fallback={null}>
<ClientProvider>
<MediaDevicesProvider>
<Sentry.ErrorBoundary
fallback={(error) => (
<ErrorPage error={error} widget={widget} />
)}
>
<DisconnectedBanner />
<Routes>
<SentryRoute path="/" element={<HomePage />} />
<SentryRoute path="/login" element={<LoginPage />} />
<SentryRoute
path="/register"
element={<RegisterPage />}
/>
<SentryRoute path="*" element={<RoomPage />} />
</Routes>
</Sentry.ErrorBoundary>
<ProcessorProvider>
<Sentry.ErrorBoundary
fallback={(error) => (
<ErrorPage error={error} widget={widget} />
)}
>
<DisconnectedBanner />
<Routes>
<SentryRoute path="/" element={<HomePage />} />
<SentryRoute path="/login" element={<LoginPage />} />
<SentryRoute
path="/register"
element={<RegisterPage />}
/>
<SentryRoute path="*" element={<RoomPage />} />
</Routes>
</Sentry.ErrorBoundary>
</ProcessorProvider>
</MediaDevicesProvider>
</ClientProvider>
</Suspense>

2
src/Avatar.test.tsx
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, expect, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import { type FC, type PropsWithChildren } from "react";
import { ClientContextProvider } from "./ClientContext";

2
src/Avatar.tsx
View File

@@ -13,7 +13,7 @@ import {
useEffect,
} from "react";
import { Avatar as CompoundAvatar } from "@vector-im/compound-web";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import { useClientState } from "./ClientContext";

6
src/ClientContext.tsx
View File

@@ -17,9 +17,9 @@ import {
type JSX,
} from "react";
import { useNavigate } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { type ISyncStateData, type SyncState } from "matrix-js-sdk/src/sync";
import { ClientEvent, type MatrixClient } from "matrix-js-sdk/src/client";
import { logger } from "matrix-js-sdk/lib/logger";
import { type ISyncStateData, type SyncState } from "matrix-js-sdk/lib/sync";
import { ClientEvent, type MatrixClient } from "matrix-js-sdk";
import type { WidgetApi } from "matrix-widget-api";
import { ErrorPage } from "./FullScreenView";

2
src/ErrorView.tsx
View File

@@ -15,7 +15,7 @@ import {
type ReactElement,
} from "react";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { RageshakeButton } from "./settings/RageshakeButton";
import styles from "./ErrorView.module.css";

2
src/FullScreenView.tsx
View File

@@ -9,7 +9,7 @@ import { type FC, type ReactElement, type ReactNode, useEffect } from "react";
import classNames from "classnames";
import { useTranslation } from "react-i18next";
import * as Sentry from "@sentry/react";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import { Header, HeaderLogo, LeftNav, RightNav } from "./Header";

2
src/IndexedDBWorker.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { IndexedDBStoreWorker } from "matrix-js-sdk/src/indexeddb-worker";
import { IndexedDBStoreWorker } from "matrix-js-sdk/lib/indexeddb-worker";
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const remoteWorker = new IndexedDBStoreWorker((self as any).postMessage);

16
src/UrlParams.ts
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { useMemo } from "react";
import { useLocation } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Config } from "./config/Config";
import { type EncryptionSystem } from "./e2ee/sharedKeyManagement";
@@ -124,9 +124,15 @@ export interface UrlParams {
*/
password: string | null;
/**
* Whether we the app should use per participant keys for E2EE.
* Whether the app should use per participant keys for E2EE.
*/
perParticipantE2EE: boolean;
/**
* Whether the global JS controls for audio output devices should be enabled,
* allowing the list of output devices to be controlled by the app hosting
* Element Call.
*/
controlledAudioDevices: boolean;
/**
* Setting this flag skips the lobby and brings you in the call directly.
* In the widget this can be combined with preload to pass the device settings
@@ -173,6 +179,7 @@ export interface UrlParams {
* The Sentry DSN. This is only used in the embedded package of Element Call.
*/
sentryDsn: string | null;
/**
* The Sentry environment. This is only used in the embedded package of Element Call.
*/
@@ -281,6 +288,11 @@ export const getUrlParams = (
fontScale: Number.isNaN(fontScale) ? null : fontScale,
allowIceFallback: parser.getFlagParam("allowIceFallback"),
perParticipantE2EE: parser.getFlagParam("perParticipantE2EE"),
controlledAudioDevices: parser.getFlagParam(
"controlledAudioDevices",
// the deprecated property name
parser.getFlagParam("controlledMediaDevices"),
),
skipLobby: parser.getFlagParam(
"skipLobby",
isWidget && intent === UserIntent.StartNewCall,

2
src/UserMenu.tsx
View File

@@ -119,7 +119,7 @@ export const UserMenu: FC<Props> = ({
key={key}
Icon={Icon}
label={label}
data-test-id={dataTestid}
data-testid={dataTestid}
onSelect={() => onAction(key)}
/>
))}

2
src/UserMenuContainer.tsx
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type FC, useCallback, useState } from "react";
import { useNavigate, useLocation } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { useClientLegacy } from "./ClientContext";
import { useProfile } from "./profile/useProfile";

4
src/analytics/PosthogAnalytics.ts
View File

@@ -10,8 +10,8 @@ import posthog, {
type PostHog,
type Properties,
} from "posthog-js";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixClient } from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixClient } from "matrix-js-sdk";
import { type Subscription } from "rxjs";
import { widget } from "../widget";

4
src/analytics/PosthogEvents.ts
View File

@@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details.
*/
import { type DisconnectReason } from "livekit-client";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import {
type IPosthogEvent,

2
src/analytics/PosthogSpanProcessor.ts
View File

@@ -11,7 +11,7 @@ import {
type Span,
} from "@opentelemetry/sdk-trace-base";
import { hrTimeToMilliseconds } from "@opentelemetry/core";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { PosthogAnalytics } from "./PosthogAnalytics";

4
src/analytics/RageshakeSpanProcessor.ts
View File

@@ -107,13 +107,13 @@ export class RageshakeSpanProcessor implements SpanProcessor {
startTime,
duration,
references:
span.parentSpanId === undefined
span.parentSpanContext?.spanId === undefined
? []
: [
{
refType: "CHILD_OF",
traceID: traceId,
spanID: span.parentSpanId,
spanID: span.parentSpanContext?.spanId,
},
],
tags: dumpAttributes(span.attributes),

10
src/auth/RegisterPage.tsx
View File

@@ -16,9 +16,9 @@ import {
} from "react";
import { useNavigate, useLocation } from "react-router-dom";
import { captureException } from "@sentry/react";
import { sleep } from "matrix-js-sdk/src/utils";
import { sleep } from "matrix-js-sdk/lib/utils";
import { Trans, useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Button, Text } from "@vector-im/compound-web";
import { FieldRow, InputField, ErrorMessage } from "../input/Input";
@@ -204,7 +204,7 @@ export const RegisterPage: FC = () => {
/>
</FieldRow>
<Text size="sm">
<Trans i18nKey="recaptcha_caption">
<Trans i18nKey="recaptcha_ssla_caption">
This site is protected by ReCAPTCHA and the Google{" "}
<ExternalLink href="https://www.google.com/policies/privacy/">
Privacy Policy
@@ -216,8 +216,8 @@ export const RegisterPage: FC = () => {
apply.
<br />
By clicking "Register", you agree to our{" "}
<ExternalLink href={Config.get().eula}>
End User Licensing Agreement (EULA)
<ExternalLink href={Config.get().ssla}>
Software and Services License Agreement (SSLA)
</ExternalLink>
</Trans>
</Text>

4
src/auth/useInteractiveLogin.ts
View File

@@ -6,12 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
import { useCallback } from "react";
import { InteractiveAuth } from "matrix-js-sdk/src/interactive-auth";
import { InteractiveAuth } from "matrix-js-sdk";
import {
createClient,
type LoginResponse,
type MatrixClient,
} from "matrix-js-sdk/src/matrix";
} from "matrix-js-sdk";
import { initClient } from "../utils/matrix";
import { type Session } from "../ClientContext";

6
src/auth/useInteractiveRegistration.ts
View File

@@ -6,13 +6,13 @@ Please see LICENSE in the repository root for full details.
*/
import { useState, useEffect, useCallback, useRef } from "react";
import { InteractiveAuth } from "matrix-js-sdk/src/interactive-auth";
import { InteractiveAuth } from "matrix-js-sdk";
import {
createClient,
type MatrixClient,
type RegisterResponse,
} from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { initClient } from "../utils/matrix";
import { type Session } from "../ClientContext";

4
src/auth/useRecaptcha.ts
View File

@@ -6,9 +6,9 @@ Please see LICENSE in the repository root for full details.
*/
import { useEffect, useCallback, useRef, useState } from "react";
import { secureRandomString } from "matrix-js-sdk/src/randomstring";
import { secureRandomString } from "matrix-js-sdk/lib/randomstring";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { translatedError } from "../TranslatedError";
declare global {

2
src/auth/useRegisterPasswordlessUser.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { useCallback } from "react";
import { secureRandomString } from "matrix-js-sdk/src/randomstring";
import { secureRandomString } from "matrix-js-sdk/lib/randomstring";
import { useClient } from "../ClientContext";
import { useInteractiveRegistration } from "../auth/useInteractiveRegistration";

2
src/button/ReactionToggleButton.test.tsx
View File

@@ -10,7 +10,7 @@ import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { userEvent } from "@testing-library/user-event";
import { type ReactNode } from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { ReactionToggleButton } from "./ReactionToggleButton";
import { ElementCallReactionEventType } from "../reactions";

2
src/button/ReactionToggleButton.tsx
View File

@@ -22,7 +22,7 @@ import {
useState,
} from "react";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import classNames from "classnames";
import { useObservableState } from "observable-hooks";
import { map } from "rxjs";

8
src/config/ConfigOptions.ts
View File

@@ -77,9 +77,9 @@ export interface ConfigOptions {
};
/**
* A link to the end-user license agreement (EULA)
* A link to the software and services license agreement (SSLA)
*/
eula?: string;
ssla?: string;
media_devices?: {
/**
@@ -134,7 +134,7 @@ export interface ResolvedConfigOptions extends ConfigOptions {
server_name: string;
};
};
eula: string;
ssla: string;
media_devices: {
enable_audio: boolean;
enable_video: boolean;
@@ -152,7 +152,7 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
features: {
feature_use_device_session_member_events: true,
},
eula: "https://static.element.io/legal/online-EULA.pdf",
ssla: "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
media_devices: {
enable_audio: true,
enable_video: true,

72
src/controls.ts
View File

@@ -5,15 +5,55 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { Subject } from "rxjs";
import { BehaviorSubject, Subject } from "rxjs";
export interface Controls {
canEnterPip: () => boolean;
enablePip: () => void;
disablePip: () => void;
canEnterPip(): boolean;
enablePip(): void;
disablePip(): void;
/** @deprecated use setAvailableAudioDevices instead*/
setAvailableOutputDevices(devices: OutputDevice[]): void;
setAvailableAudioDevices(devices: OutputDevice[]): void;
/** @deprecated use setAudioDevice instead*/
setOutputDevice(id: string): void;
setAudioDevice(id: string): void;
/** @deprecated use onAudioDeviceSelect instead*/
onOutputDeviceSelect?: (id: string) => void;
onAudioDeviceSelect?: (id: string) => void;
/** @deprecated use setAudioEnabled instead*/
setOutputEnabled(enabled: boolean): void;
setAudioEnabled(enabled: boolean): void;
/** @deprecated use showNativeAudioDevicePicker instead*/
showNativeOutputDevicePicker?: () => void;
showNativeAudioDevicePicker?: () => void;
}
export interface OutputDevice {
id: string;
name: string;
forEarpiece?: boolean;
isEarpiece?: boolean;
isSpeaker?: boolean;
isExternalHeadset?: boolean;
}
/**
* If pipMode is enabled, EC will render a adapted call view layout.
*/
export const setPipEnabled$ = new Subject<boolean>();
// BehaviorSubject since the client might set this before we have subscribed (GroupCallView still in "loading" state)
// We want the devices that have been set during loading to be available immediately once loaded.
export const availableOutputDevices$ = new BehaviorSubject<OutputDevice[]>([]);
// BehaviorSubject since the client might set this before we have subscribed (GroupCallView still in "loading" state)
// We want the device that has been set during loading to be available immediately once loaded.
export const outputDevice$ = new BehaviorSubject<string | undefined>(undefined);
/**
* This allows the os to mute the call if the user
* presses the volume down button when it is at the minimum volume.
*
* This should also be used to display a darkened overlay screen letting the user know that audio is muted.
*/
export const setAudioEnabled$ = new Subject<boolean>();
window.controls = {
canEnterPip(): boolean {
@@ -27,4 +67,28 @@ window.controls = {
if (!setPipEnabled$.observed) throw new Error("No call is running");
setPipEnabled$.next(false);
},
setAvailableAudioDevices(devices: OutputDevice[]): void {
availableOutputDevices$.next(devices);
},
setAudioDevice(id: string): void {
outputDevice$.next(id);
},
setAudioEnabled(enabled: boolean): void {
if (!setAudioEnabled$.observed)
throw new Error(
"Output controls are disabled. No setAudioEnabled$ observer",
);
setAudioEnabled$.next(enabled);
},
// wrappers for the deprecated controls fields
setOutputEnabled(enabled: boolean): void {
this.setAudioEnabled(enabled);
},
setAvailableOutputDevices(devices: OutputDevice[]): void {
this.setAvailableAudioDevices(devices);
},
setOutputDevice(id: string): void {
this.setAudioDevice(id);
},
};

2
src/e2ee/matrixKeyProvider.test.ts
View File

@@ -9,7 +9,7 @@ import { describe, expect, test, vi } from "vitest";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
} from "matrix-js-sdk/lib/matrixrtc";
import { KeyProviderEvent } from "livekit-client";
import { MatrixKeyProvider } from "./matrixKeyProvider";

45
src/e2ee/matrixKeyProvider.ts
View File

@@ -5,18 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BaseKeyProvider, createKeyMaterialFromBuffer } from "livekit-client";
import { logger } from "matrix-js-sdk/src/logger";
import { BaseKeyProvider } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
} from "matrix-js-sdk/lib/matrixrtc";
export class MatrixKeyProvider extends BaseKeyProvider {
private rtcSession?: MatrixRTCSession;
public constructor() {
super({ ratchetWindowSize: 0, keyringSize: 256 });
super({ ratchetWindowSize: 10, keyringSize: 256 });
}
public setRTCSession(rtcSession: MatrixRTCSession): void {
@@ -44,20 +44,29 @@ export class MatrixKeyProvider extends BaseKeyProvider {
encryptionKeyIndex: number,
participantId: string,
): void => {
createKeyMaterialFromBuffer(encryptionKey).then(
(keyMaterial) => {
this.onSetEncryptionKey(keyMaterial, participantId, encryptionKeyIndex);
crypto.subtle
.importKey("raw", encryptionKey, "HKDF", false, [
"deriveBits",
"deriveKey",
])
.then(
(keyMaterial) => {
this.onSetEncryptionKey(
keyMaterial,
participantId,
encryptionKeyIndex,
);
logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
);
},
(e) => {
logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
e,
);
},
);
logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
);
},
(e) => {
logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
e,
);
},
);
};
}

2
src/e2ee/sharedKeyManagement.ts
View File

@@ -21,7 +21,7 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
const useInternalRoomSharedKey = (roomId: string): string | null => {
const key = getRoomSharedKeyLocalStorageKey(roomId);
const roomSharedKey = useLocalStorage(key)[0];
const [roomSharedKey] = useLocalStorage(key);
return roomSharedKey;
};

2
src/grid/Grid.tsx
View File

@@ -32,7 +32,7 @@ import {
} from "react";
import useMeasure from "react-use-measure";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { useObservableEagerState } from "observable-hooks";
import { fromEvent, map, startWith } from "rxjs";

2
src/home/CallList.test.tsx
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { render, type RenderResult } from "@testing-library/react";
import { type MatrixClient } from "matrix-js-sdk/src/matrix";
import { type MatrixClient } from "matrix-js-sdk";
import { MemoryRouter } from "react-router-dom";
import { describe, expect, it } from "vitest";

4
src/home/CallList.tsx
View File

@@ -6,9 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { Link } from "react-router-dom";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type RoomMember } from "matrix-js-sdk/src/models/room-member";
import { type Room } from "matrix-js-sdk/src/models/room";
import { type RoomMember, type Room, type MatrixClient } from "matrix-js-sdk";
import { type FC, useCallback, type MouseEvent, useState } from "react";
import { useTranslation } from "react-i18next";
import { IconButton, Text } from "@vector-im/compound-web";

4
src/home/RegisteredView.tsx
View File

@@ -12,10 +12,10 @@ import {
type FormEventHandler,
type FC,
} from "react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import { useTranslation } from "react-i18next";
import { Heading, Text } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Button } from "@vector-im/compound-web";
import { useNavigate } from "react-router-dom";

10
src/home/UnauthenticatedView.tsx
View File

@@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details.
*/
import { type FC, useCallback, useState, type FormEventHandler } from "react";
import { secureRandomString } from "matrix-js-sdk/src/randomstring";
import { secureRandomString } from "matrix-js-sdk/lib/randomstring";
import { Trans, useTranslation } from "react-i18next";
import { Button, Heading, Text } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { useNavigate } from "react-router-dom";
import { useClient } from "../ClientContext";
@@ -185,10 +185,10 @@ export const UnauthenticatedView: FC = () => {
</Text>
)}
<Text size="sm" className={styles.notice}>
<Trans i18nKey="unauthenticated_view_eula_caption">
<Trans i18nKey="unauthenticated_view_ssla_caption">
By clicking "Go", you agree to our{" "}
<ExternalLink href={Config.get().eula}>
End User Licensing Agreement (EULA)
<ExternalLink href={Config.get().ssla}>
Software and Services License Agreement (SSLA)
</ExternalLink>
</Trans>
</Text>

21
src/home/useGroupCallRooms.ts
View File

@@ -5,14 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type Room, RoomEvent } from "matrix-js-sdk/src/models/room";
import { type RoomMember } from "matrix-js-sdk/src/models/room-member";
import {
type MatrixClient,
type RoomMember,
type Room,
RoomEvent,
EventTimeline,
EventType,
JoinRule,
KnownMembership,
} from "matrix-js-sdk";
import { useState, useEffect } from "react";
import { EventTimeline, EventType, JoinRule } from "matrix-js-sdk/src/matrix";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
import { KnownMembership } from "matrix-js-sdk/src/types";
import {
MatrixRTCSessionManagerEvents,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { getKeyForRoom } from "../e2ee/sharedKeyManagement";

7
src/initializer.tsx
View File

@@ -14,7 +14,7 @@ import i18n, {
import { initReactI18next } from "react-i18next";
import LanguageDetector from "i18next-browser-languagedetector";
import * as Sentry from "@sentry/react";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { shouldPolyfill as shouldPolyfillSegmenter } from "@formatjs/intl-segmenter/should-polyfill";
import { shouldPolyfill as shouldPolyfillDurationFormat } from "@formatjs/intl-durationformat/should-polyfill";
import {
@@ -136,6 +136,11 @@ export class Initializer {
lookup: () => getUrlParams().lang ?? undefined,
});
// Synchronise the HTML lang attribute with the i18next language
i18n.on("languageChanged", (lng) => {
document.documentElement.lang = lng;
});
await i18n
.use(Backend)
.use(languageDetector)

80
src/livekit/BlurBackgroundTransformer.ts Normal file
View File

@@ -0,0 +1,80 @@
/*
Copyright 2024-2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
BackgroundTransformer,
VideoTransformer,
type VideoTransformerInitOptions,
} from "@livekit/track-processors";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
interface WasmFileset {
/** The path to the Wasm loader script. */
wasmLoaderPath: string;
/** The path to the Wasm binary. */
wasmBinaryPath: string;
}
// The MediaPipe package, by default, ships some alternative versions of the
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
// in WASM is actually fine by our support policy, so we include just the SIMD
// versions.
// It's really not ideal that we have to reference these internal files from
// MediaPipe and depend on node_modules having this specific structure. It's
// easy to see this breaking if our dependencies changed and MediaPipe were
// no longer hoisted, or if we switched to another dependency loader such as
// Yarn PnP.
// https://github.com/google-ai-edge/mediapipe/issues/5961
const wasmFileset: WasmFileset = {
wasmLoaderPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
import.meta.url,
).href,
wasmBinaryPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
import.meta.url,
).href,
};
/**
* Track processor that applies effects such as blurring to a user's background.
*
* This is just like LiveKit's prebuilt BackgroundTransformer except that it
* loads the segmentation models from our own bundle rather than as an external
* resource fetched from the public internet.
*/
export class BlurBackgroundTransformer extends BackgroundTransformer {
public async init({
outputCanvas,
inputElement: inputVideo,
}: VideoTransformerInitOptions): Promise<void> {
// Call super.super.init() since we're totally replacing the init method of
// BackgroundTransformer here, rather than extending it
await VideoTransformer.prototype.init.call(this, {
outputCanvas,
inputElement: inputVideo,
});
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
baseOptions: {
modelAssetPath,
delegate: "GPU",
...this.options.segmenterOptions,
},
canvas: this.canvas,
runningMode: "VIDEO",
outputCategoryMask: true,
outputConfidenceMasks: false,
});
if (this.options.blurRadius) {
this.gl?.setBlurRadius(this.options.blurRadius);
}
}
}

104
src/livekit/MatrixAudioRenderer.test.tsx Normal file
View File

@@ -0,0 +1,104 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { render } from "@testing-library/react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import {
getTrackReferenceId,
type TrackReference,
} from "@livekit/components-core";
import { type RemoteAudioTrack } from "livekit-client";
import { type ReactNode } from "react";
import { useTracks } from "@livekit/components-react";
import { testAudioContext } from "../useAudioContext.test";
import * as MediaDevicesContext from "./MediaDevicesContext";
import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
import { mockTrack } from "../utils/test";
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
beforeEach(() => {
vi.stubGlobal("AudioContext", TestAudioContextConstructor);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
vi.mock("@livekit/components-react", async (importOriginal) => {
return {
...(await importOriginal()),
AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
return (
<audio data-testid={"audio"}>
{getTrackReferenceId(props.trackRef)}
</audio>
);
},
useTracks: vi.fn(),
};
});
const tracks = [mockTrack("test:123")];
vi.mocked(useTracks).mockReturnValue(tracks);
it("should render for member", () => {
const { container, queryAllByTestId } = render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(1);
});
it("should not render without member", () => {
const { container, queryAllByTestId } = render(
<MatrixAudioRenderer
members={[{ sender: "othermember", deviceId: "123" }] as CallMembership[]}
/>,
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(0);
});
it("should not setup audioContext gain and pan if there is no need to.", () => {
render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]);
expect(testAudioContext.gain.gain.value).toEqual(1);
expect(testAudioContext.pan.pan.value).toEqual(0);
});
it("should setup audioContext gain and pan", () => {
vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
pan: 1,
volume: 0.1,
});
render(
<MatrixAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>,
);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalled();
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled();
expect(testAudioContext.gain.gain.value).toEqual(0.1);
expect(testAudioContext.pan.pan.value).toEqual(1);
});

212
src/livekit/MatrixAudioRenderer.tsx Normal file
View File

@@ -0,0 +1,212 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { getTrackReferenceId } from "@livekit/components-core";
import { type RemoteAudioTrack, Track } from "livekit-client";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import {
useTracks,
AudioTrack,
type AudioTrackProps,
} from "@livekit/components-react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { useEarpieceAudioConfig } from "./MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
export interface MatrixAudioRendererProps {
/**
* The list of participants to render audio for.
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session.
*/
members: CallMembership[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
* If set to `true`, the server will stop sending audio track data to the client.
*/
muted?: boolean;
}
/**
* The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
* It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
* @example
* ```tsx
* <LiveKitRoom>
* <MatrixAudioRenderer />
* </LiveKitRoom>
* ```
* @public
*/
export function MatrixAudioRenderer({
members,
muted,
}: MatrixAudioRendererProps): ReactNode {
const validIdentities = useMemo(
() =>
new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
[members],
);
const loggedInvalidIdentities = useRef(new Set<string>());
/**
* Log an invalid livekit track identity.
* A invalid identity is one that does not match any of the matrix rtc members.
*
* @param identity The identity of the track that is invalid
* @param validIdentities The list of valid identities
*/
const logInvalid = (identity: string, validIdentities: Set<string>): void => {
if (loggedInvalidIdentities.current.has(identity)) return;
logger.warn(
`Audio track ${identity} has no matching matrix call member`,
`current members: ${Array.from(validIdentities.values())}`,
`track will not get rendered`,
);
loggedInvalidIdentities.current.add(identity);
};
const tracks = useTracks(
[
Track.Source.Microphone,
Track.Source.ScreenShareAudio,
Track.Source.Unknown,
],
{
updateOnlyOn: [],
onlySubscribed: true,
},
).filter((ref) => {
const isValid = validIdentities?.has(ref.participant.identity);
if (!isValid && !ref.participant.isLocal)
logInvalid(ref.participant.identity, validIdentities);
return (
!ref.participant.isLocal &&
ref.publication.kind === Track.Kind.Audio &&
isValid
);
});
// This component is also (in addition to the "only play audio for connected members" logic above)
// responsible for mimicking earpiece audio on iPhones.
// The Safari audio devices enumeration does not expose an earpiece audio device.
// We alternatively use the audioContext pan node to only use one of the stereo channels.
// This component does get additionally complicated because of a Safari bug.
// (see: https://bugs.webkit.org/show_bug.cgi?id=251532
// and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
// and https://bugs.webkit.org/show_bug.cgi?id=231105)
//
// AudioContext gets stopped if the webview gets moved into the background.
// Once the phone is in standby audio playback will stop.
// So we can only use the pan trick only works is the phone is not in standby.
// If earpiece mode is not used we do not use audioContext to allow standby playback.
// shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
const shouldUseAudioContext = stereoPan !== 0;
// initialize the potentially used audio context.
const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
undefined,
);
useEffect(() => {
const ctx = new AudioContext();
setAudioContext(ctx);
return (): void => {
void ctx.close();
};
}, []);
const audioNodes = useMemo(
() => ({
gain: audioContext?.createGain(),
pan: audioContext?.createStereoPanner(),
}),
[audioContext],
);
// Simple effects to update the gain and pan node based on the props
useEffect(() => {
if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
}, [audioNodes.pan, stereoPan]);
useEffect(() => {
if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
}, [audioNodes.gain, volumeFactor]);
return (
// We add all audio elements into one <div> for the browser developer tool experience/tidyness.
<div style={{ display: "none" }}>
{tracks.map((trackRef) => (
<AudioTrackWithAudioNodes
key={getTrackReferenceId(trackRef)}
trackRef={trackRef}
muted={muted}
audioContext={shouldUseAudioContext ? audioContext : undefined}
audioNodes={audioNodes}
/>
))}
</div>
);
}
interface StereoPanAudioTrackProps {
muted?: boolean;
audioContext?: AudioContext;
audioNodes: {
gain?: GainNode;
pan?: StereoPannerNode;
};
}
/**
* This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
* It main purpose is to remount the AudioTrack component when switching from
* audiooContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param param0
* @returns
*/
function AudioTrackWithAudioNodes({
trackRef,
muted,
audioContext,
audioNodes,
...props
}: StereoPanAudioTrackProps &
AudioTrackProps &
React.RefAttributes<HTMLAudioElement>): ReactNode {
// This is used to unmount/remount the AudioTrack component.
// Mounting needs to happen after the audioContext is set.
// (adding the audio context when already mounted did not work outside strict mode)
const [trackReady, setTrackReady] = useReactiveState(
() => false,
// We only want the track to reset once both (audioNodes and audioContext) are set.
// for unsetting the audioContext its enough if one of the the is undefined.
[audioContext && audioNodes],
);
useEffect(() => {
if (!trackRef || trackReady) return;
const track = trackRef.publication.track as RemoteAudioTrack;
const useContext = audioContext && audioNodes.gain && audioNodes.pan;
track.setAudioContext(useContext ? audioContext : undefined);
track.setWebAudioPlugins(
useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
);
setTrackReady(true);
}, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
return (
trackReady && <AudioTrack trackRef={trackRef} muted={muted} {...props} />
);
}

270
src/livekit/MediaDevicesContext.tsx
View File

@@ -1,5 +1,5 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2023-2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -17,29 +17,42 @@ import {
type JSX,
} from "react";
import { createMediaDeviceObserver } from "@livekit/components-core";
import { map, startWith } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
import { combineLatest, map, startWith } from "rxjs";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/lib/logger";
import {
useSetting,
audioInput as audioInputSetting,
audioOutput as audioOutputSetting,
videoInput as videoInputSetting,
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
type Setting,
} from "../settings/settings";
import { outputDevice$, availableOutputDevices$ } from "../controls";
import { useUrlParams } from "../UrlParams";
// This hardcoded id is used in EX ios! It can only be changed in coordination with
// the ios swift team.
export const EARPIECE_CONFIG_ID = "earpiece-id";
export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number }
| { type: "earpiece" }
| { type: "default"; name: string | null };
export interface MediaDevice {
export interface MediaDeviceHandle {
/**
* A map from available device IDs to labels.
*/
available: Map<string, DeviceLabel>;
selectedId: string | undefined;
/**
* An additional device configuration that makes us use only one channel of the
* output device and a reduced volume.
*/
useAsEarpiece: boolean | undefined;
/**
* The group ID of the selected device.
*/
@@ -50,23 +63,69 @@ export interface MediaDevice {
select: (deviceId: string) => void;
}
export interface MediaDevices {
audioInput: MediaDevice;
audioOutput: MediaDevice;
videoInput: MediaDevice;
interface InputDevices {
audioInput: MediaDeviceHandle;
videoInput: MediaDeviceHandle;
startUsingDeviceNames: () => void;
stopUsingDeviceNames: () => void;
usingNames: boolean;
}
function useMediaDevice(
export interface MediaDevices extends Omit<InputDevices, "usingNames"> {
audioOutput: MediaDeviceHandle;
}
/**
* An observable that represents if we should display the devices menu for iOS.
* This implies the following
* - hide any input devices (they do not work anyhow on ios)
* - Show a button to show the native output picker instead.
* - Only show the earpiece toggle option if the earpiece is available:
* `availableOutputDevices$.includes((d)=>d.forEarpiece)`
*/
export const iosDeviceMenu$ = alwaysShowIphoneEarpieceSetting.value$.pipe(
map((v) => v || navigator.userAgent.includes("iPhone")),
);
function useSelectedId(
available: Map<string, DeviceLabel>,
preferredId: string | undefined,
): string | undefined {
return useMemo(() => {
if (available.size) {
// If the preferred device is available, use it. Or if every available
// device ID is falsy, the browser is probably just being paranoid about
// fingerprinting and we should still try using the preferred device.
// Worst case it is not available and the browser will gracefully fall
// back to some other device for us when requesting the media stream.
// Otherwise, select the first available device.
return (preferredId !== undefined && available.has(preferredId)) ||
(available.size === 1 && available.has(""))
? preferredId
: available.keys().next().value;
}
return undefined;
}, [available, preferredId]);
}
/**
* Hook to get access to a mediaDevice handle for a kind. This allows to list
* the available devices, read and set the selected device.
* @param kind Audio input, output or video output.
* @param setting The setting this handle's selection should be synced with.
* @param usingNames If the hook should query device names for the associated
* list.
* @returns A handle for the chosen kind.
*/
function useMediaDeviceHandle(
kind: MediaDeviceKind,
setting: Setting<string | undefined>,
usingNames: boolean,
): MediaDevice {
// Make sure we don't needlessly reset to a device observer without names,
// once permissions are already given
): MediaDeviceHandle {
const hasRequestedPermissions = useRef(false);
const requestPermissions = usingNames || hasRequestedPermissions.current;
// Make sure we don't needlessly reset to a device observer without names,
// once permissions are already given
hasRequestedPermissions.current ||= usingNames;
// We use a bare device observer here rather than one of the fancy device
@@ -102,11 +161,13 @@ function useMediaDevice(
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
// We also create this if we do not have any available devices, so that
// we can use the default or the earpiece.
if (
kind === "audiooutput" &&
available.size &&
!available.has("") &&
!available.has("default")
!available.has("default") &&
available.size
)
available = new Map([
["", { type: "default", name: availableRaw[0]?.label || null }],
@@ -118,26 +179,13 @@ function useMediaDevice(
return available;
}),
),
[kind, deviceObserver$],
[deviceObserver$, kind],
),
);
const [preferredId, select] = useSetting(setting);
const selectedId = useMemo(() => {
if (available.size) {
// If the preferred device is available, use it. Or if every available
// device ID is falsy, the browser is probably just being paranoid about
// fingerprinting and we should still try using the preferred device.
// Worst case it is not available and the browser will gracefully fall
// back to some other device for us when requesting the media stream.
// Otherwise, select the first available device.
return (preferredId !== undefined && available.has(preferredId)) ||
(available.size === 1 && available.has(""))
? preferredId
: available.keys().next().value;
}
return undefined;
}, [available, preferredId]);
const selectedId = useSelectedId(available, preferredId);
const selectedGroupId = useObservableEagerState(
useMemo(
() =>
@@ -155,6 +203,7 @@ function useMediaDevice(
() => ({
available,
selectedId,
useAsEarpiece: false,
selectedGroupId,
select,
}),
@@ -162,12 +211,14 @@ function useMediaDevice(
);
}
export const deviceStub: MediaDevice = {
export const deviceStub: MediaDeviceHandle = {
available: new Map(),
selectedId: undefined,
selectedGroupId: undefined,
select: () => {},
useAsEarpiece: false,
};
export const devicesStub: MediaDevices = {
audioInput: deviceStub,
audioOutput: deviceStub,
@@ -178,26 +229,17 @@ export const devicesStub: MediaDevices = {
export const MediaDevicesContext = createContext<MediaDevices>(devicesStub);
interface Props {
children: JSX.Element;
}
export const MediaDevicesProvider: FC<Props> = ({ children }) => {
function useInputDevices(): InputDevices {
// Counts the number of callers currently using device names.
const [numCallersUsingNames, setNumCallersUsingNames] = useState(0);
const usingNames = numCallersUsingNames > 0;
const audioInput = useMediaDevice(
const audioInput = useMediaDeviceHandle(
"audioinput",
audioInputSetting,
usingNames,
);
const audioOutput = useMediaDevice(
"audiooutput",
audioOutputSetting,
usingNames,
);
const videoInput = useMediaDevice(
const videoInput = useMediaDeviceHandle(
"videoinput",
videoInputSetting,
usingNames,
@@ -212,17 +254,52 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
[setNumCallersUsingNames],
);
return {
audioInput,
videoInput,
startUsingDeviceNames,
stopUsingDeviceNames,
usingNames,
};
}
interface Props {
children: JSX.Element;
}
export const MediaDevicesProvider: FC<Props> = ({ children }) => {
const {
audioInput,
videoInput,
startUsingDeviceNames,
stopUsingDeviceNames,
usingNames,
} = useInputDevices();
const { controlledAudioDevices } = useUrlParams();
const webViewAudioOutput = useMediaDeviceHandle(
"audiooutput",
audioOutputSetting,
usingNames,
);
const controlledAudioOutput = useControlledOutput();
const context: MediaDevices = useMemo(
() => ({
audioInput,
audioOutput,
audioOutput: controlledAudioDevices
? controlledAudioOutput
: webViewAudioOutput,
videoInput,
startUsingDeviceNames,
stopUsingDeviceNames,
}),
[
audioInput,
audioOutput,
controlledAudioDevices,
controlledAudioOutput,
webViewAudioOutput,
videoInput,
startUsingDeviceNames,
stopUsingDeviceNames,
@@ -236,6 +313,80 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
);
};
function useControlledOutput(): MediaDeviceHandle {
const { available } = useObservableEagerState(
useObservable(() => {
const outputDeviceData$ = availableOutputDevices$.pipe(
map((devices) => {
const deviceForEarpiece = devices.find((d) => d.forEarpiece);
const deviceMapTuple: [string, DeviceLabel][] = devices.map(
({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => {
let deviceLabel: DeviceLabel = { type: "name", name };
// if (isExternalHeadset) // Do we want this?
if (isEarpiece) deviceLabel = { type: "earpiece" };
if (isSpeaker) deviceLabel = { type: "default", name };
return [id, deviceLabel];
},
);
return {
devicesMap: new Map<string, DeviceLabel>(deviceMapTuple),
deviceForEarpiece,
};
}),
);
return combineLatest(
[outputDeviceData$, iosDeviceMenu$],
({ devicesMap, deviceForEarpiece }, iosShowEarpiece) => {
let available = devicesMap;
if (iosShowEarpiece && !!deviceForEarpiece) {
available = new Map([
...devicesMap.entries(),
[EARPIECE_CONFIG_ID, { type: "earpiece" }],
]);
}
return { available, deviceForEarpiece };
},
);
}),
);
const [preferredId, setPreferredId] = useSetting(audioOutputSetting);
useEffect(() => {
const subscription = outputDevice$.subscribe((id) => {
if (id) setPreferredId(id);
});
return (): void => subscription.unsubscribe();
}, [setPreferredId]);
const selectedId = useSelectedId(available, preferredId);
const [asEarpiece, setAsEarpiece] = useState(false);
useEffect(() => {
// Let the hosting application know which output device has been selected.
// This information is probably only of interest if the earpiece mode has been
// selected - for example, Element X iOS listens to this to determine whether it
// should enable the proximity sensor.
if (selectedId) {
window.controls.onAudioDeviceSelect?.(selectedId);
// Call deprecated method for backwards compatibility.
window.controls.onOutputDeviceSelect?.(selectedId);
}
setAsEarpiece(selectedId === EARPIECE_CONFIG_ID);
}, [selectedId]);
return useMemo(
() => ({
available: available,
selectedId,
selectedGroupId: undefined,
select: setPreferredId,
useAsEarpiece: asEarpiece,
}),
[available, selectedId, setPreferredId, asEarpiece],
);
}
export const useMediaDevices = (): MediaDevices =>
useContext(MediaDevicesContext);
@@ -255,3 +406,30 @@ export const useMediaDeviceNames = (
return context.stopUsingDeviceNames;
}
}, [context, enabled]);
/**
* A convenience hook to get the audio node configuration for the earpiece.
* It will check the `useAsEarpiece` of the `audioOutput` device and return
* the appropriate pan and volume values.
*
* @returns pan and volume values for the earpiece audio node configuration.
*/
export const useEarpieceAudioConfig = (): {
pan: number;
volume: number;
} => {
const { audioOutput } = useMediaDevices();
// We use only the right speaker (pan = 1) for the earpiece.
// This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone)
const pan = useMemo(
() => (audioOutput.useAsEarpiece ? 1 : 0),
[audioOutput.useAsEarpiece],
);
// We also do lower the volume by a factor of 10 to optimize for the usecase where
// a user is holding the phone to their ear.
const volume = useMemo(
() => (audioOutput.useAsEarpiece ? 0.1 : 1),
[audioOutput.useAsEarpiece],
);
return { pan, volume };
};

84
src/livekit/TrackProcessorContext.tsx Normal file
View File

@@ -0,0 +1,84 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
ProcessorWrapper,
supportsBackgroundProcessors,
type BackgroundOptions,
} from "@livekit/track-processors";
import { createContext, type FC, useContext, useEffect, useMemo } from "react";
import { type LocalVideoTrack } from "livekit-client";
import {
backgroundBlur as backgroundBlurSettings,
useSetting,
} from "../settings/settings";
import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
type ProcessorState = {
supported: boolean | undefined;
processor: undefined | ProcessorWrapper<BackgroundOptions>;
};
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
export function useTrackProcessor(): ProcessorState {
const state = useContext(ProcessorContext);
if (state === undefined)
throw new Error(
"useTrackProcessor must be used within a ProcessorProvider",
);
return state;
}
export const useTrackProcessorSync = (
videoTrack: LocalVideoTrack | null,
): void => {
const { processor } = useTrackProcessor();
useEffect(() => {
if (!videoTrack) return;
if (processor && !videoTrack.getProcessor()) {
void videoTrack.setProcessor(processor);
}
if (!processor && videoTrack.getProcessor()) {
void videoTrack.stopProcessor();
}
}, [processor, videoTrack]);
};
interface Props {
children: JSX.Element;
}
export const ProcessorProvider: FC<Props> = ({ children }) => {
// The setting the user wants to have
const [blurActivated] = useSetting(backgroundBlurSettings);
const supported = useMemo(() => supportsBackgroundProcessors(), []);
const blur = useMemo(
() =>
new ProcessorWrapper(
new BlurBackgroundTransformer({ blurRadius: 15 }),
"background-blur",
),
[],
);
// This is the actual state exposed through the context
const processorState = useMemo(
() => ({
supported,
processor: supported && blurActivated ? blur : undefined,
}),
[supported, blurActivated, blur],
);
return (
<ProcessorContext.Provider value={processorState}>
{children}
</ProcessorContext.Provider>
);
};

8
src/livekit/openIDSFU.ts
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { useEffect, useState } from "react";
import { type LivekitFocus } from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc";
import { useActiveLivekitFocus } from "../room/useActiveFocus";
import { useErrorBoundary } from "../useErrorBoundary";

113
src/livekit/useECConnectionState.test.tsx
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { type FC, useCallback, useState } from "react";
import { test, vi } from "vitest";
import { describe, expect, test, vi, vitest } from "vitest";
import {
ConnectionError,
ConnectionErrorReason,
@@ -15,6 +15,7 @@ import {
import userEvent from "@testing-library/user-event";
import { render, screen } from "@testing-library/react";
import { MemoryRouter } from "react-router-dom";
import { defer, sleep } from "matrix-js-sdk/lib/utils";
import { useECConnectionState } from "./useECConnectionState";
import { type SFUConfig } from "./openIDSFU";
@@ -57,7 +58,7 @@ test.each<[string, ConnectionError]>([
() => setSfuConfig({ url: "URL", jwt: "JWT token" }),
[],
);
useECConnectionState({}, false, mockRoom, sfuConfig);
useECConnectionState("default", false, mockRoom, sfuConfig);
return <button onClick={connect}>Connect</button>;
};
@@ -73,3 +74,111 @@ test.each<[string, ConnectionError]>([
screen.getByText("Insufficient capacity");
},
);
describe("Leaking connection prevention", () => {
function createTestComponent(mockRoom: Room): FC {
const TestComponent: FC = () => {
const [sfuConfig, setSfuConfig] = useState<SFUConfig | undefined>(
undefined,
);
const connect = useCallback(
() => setSfuConfig({ url: "URL", jwt: "JWT token" }),
[],
);
useECConnectionState("default", false, mockRoom, sfuConfig);
return <button onClick={connect}>Connect</button>;
};
return TestComponent;
}
test("Should cancel pending connections when the component is unmounted", async () => {
const connectCall = vi.fn();
const pendingConnection = defer<void>();
// let pendingDisconnection = defer<void>()
const disconnectMock = vi.fn();
const mockRoom = {
on: () => {},
off: () => {},
once: () => {},
connect: async () => {
connectCall.call(undefined);
return await pendingConnection.promise;
},
disconnect: disconnectMock,
localParticipant: {
getTrackPublication: () => {},
createTracks: () => [],
},
} as unknown as Room;
const TestComponent = createTestComponent(mockRoom);
const { unmount } = render(<TestComponent />);
const user = userEvent.setup();
await user.click(screen.getByRole("button", { name: "Connect" }));
expect(connectCall).toHaveBeenCalled();
// unmount while the connection is pending
unmount();
// resolve the pending connection
pendingConnection.resolve();
await vitest.waitUntil(
() => {
return disconnectMock.mock.calls.length > 0;
},
{
timeout: 1000,
interval: 100,
},
);
// There should be some cleaning up to avoid leaking an open connection
expect(disconnectMock).toHaveBeenCalledTimes(1);
});
test("Should cancel about to open but not yet opened connection", async () => {
const createTracksCall = vi.fn();
const pendingCreateTrack = defer<void>();
// let pendingDisconnection = defer<void>()
const disconnectMock = vi.fn();
const connectMock = vi.fn();
const mockRoom = {
on: () => {},
off: () => {},
once: () => {},
connect: connectMock,
disconnect: disconnectMock,
localParticipant: {
getTrackPublication: () => {},
createTracks: async () => {
createTracksCall.call(undefined);
await pendingCreateTrack.promise;
return [];
},
},
} as unknown as Room;
const TestComponent = createTestComponent(mockRoom);
const { unmount } = render(<TestComponent />);
const user = userEvent.setup();
await user.click(screen.getByRole("button", { name: "Connect" }));
expect(createTracksCall).toHaveBeenCalled();
// unmount while createTracks is pending
unmount();
// resolve createTracks
pendingCreateTrack.resolve();
// Yield to the event loop to let the connection attempt finish
await sleep(100);
// The operation should have been aborted before even calling connect.
expect(connectMock).not.toHaveBeenCalled();
});
});

76
src/livekit/useECConnectionState.ts
View File

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/
import {
type AudioCaptureOptions,
ConnectionError,
ConnectionState,
type LocalTrack,
@@ -15,7 +14,7 @@ import {
Track,
} from "livekit-client";
import { useCallback, useEffect, useRef, useState } from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import * as Sentry from "@sentry/react";
import { type SFUConfig, sfuConfigEquals } from "./openIDSFU";
@@ -25,6 +24,7 @@ import {
InsufficientCapacityError,
UnknownCallError,
} from "../utils/errors.ts";
import { AbortHandle } from "../utils/abortHandle.ts";
declare global {
interface Window {
@@ -59,7 +59,8 @@ async function doConnect(
livekitRoom: Room,
sfuConfig: SFUConfig,
audioEnabled: boolean,
audioOptions: AudioCaptureOptions,
initialDeviceId: string | undefined,
abortHandle: AbortHandle,
): Promise<void> {
// Always create an audio track manually.
// livekit (by default) keeps the mic track open when you mute, but if you start muted,
@@ -82,19 +83,40 @@ async function doConnect(
let preCreatedAudioTrack: LocalTrack | undefined;
try {
const audioTracks = await livekitRoom!.localParticipant.createTracks({
audio: audioOptions,
audio: { deviceId: initialDeviceId },
});
if (audioTracks.length < 1) {
logger.info("Tried to pre-create local audio track but got no tracks");
} else {
preCreatedAudioTrack = audioTracks[0];
}
// There was a yield point previously (awaiting for the track to be created) so we need to check
// if the operation was cancelled and stop connecting if needed.
if (abortHandle.isAborted()) {
logger.info(
"[Lifecycle] Signal Aborted: Pre-created audio track but connection aborted",
);
preCreatedAudioTrack?.stop();
return;
}
logger.info("Pre-created microphone track");
} catch (e) {
logger.error("Failed to pre-create microphone track", e);
}
if (!audioEnabled) await preCreatedAudioTrack?.mute();
if (!audioEnabled) {
await preCreatedAudioTrack?.mute();
// There was a yield point. Check if the operation was cancelled and stop connecting.
if (abortHandle.isAborted()) {
logger.info(
"[Lifecycle] Signal Aborted: Pre-created audio track but connection aborted",
);
preCreatedAudioTrack?.stop();
return;
}
}
// check again having awaited for the track to create
if (
@@ -107,9 +129,18 @@ async function doConnect(
return;
}
logger.info("Connecting & publishing");
logger.info("[Lifecycle] Connecting & publishing");
try {
await connectAndPublish(livekitRoom, sfuConfig, preCreatedAudioTrack, []);
if (abortHandle.isAborted()) {
logger.info(
"[Lifecycle] Signal Aborted: Connected but operation was cancelled. Force disconnect",
);
livekitRoom?.disconnect().catch((err) => {
logger.error("Failed to disconnect from SFU", err);
});
return;
}
} catch (e) {
preCreatedAudioTrack?.stop();
logger.debug("Stopped precreated audio tracks.");
@@ -137,13 +168,16 @@ async function connectAndPublish(
livekitRoom.once(RoomEvent.SignalConnected, tracker.cacheWsConnect);
try {
logger.info(`[Lifecycle] Connecting to livekit room ${sfuConfig!.url} ...`);
await livekitRoom!.connect(sfuConfig!.url, sfuConfig!.jwt, {
// Due to stability issues on Firefox we are testing the effect of different
// timeouts, and allow these values to be set through the console
peerConnectionTimeout: window.peerConnectionTimeout ?? 45000,
websocketTimeout: window.websocketTimeout ?? 45000,
});
logger.info(`[Lifecycle] ... connected to livekit room`);
} catch (e) {
logger.error("[Lifecycle] Failed to connect", e);
// LiveKit uses 503 to indicate that the server has hit its track limits.
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
// It also errors with a status code of 200 (yes, really) for room
@@ -184,7 +218,7 @@ async function connectAndPublish(
}
export function useECConnectionState(
initialAudioOptions: AudioCaptureOptions,
initialDeviceId: string | undefined,
initialAudioEnabled: boolean,
livekitRoom?: Room,
sfuConfig?: SFUConfig,
@@ -247,6 +281,22 @@ export function useECConnectionState(
const currentSFUConfig = useRef(Object.assign({}, sfuConfig));
// Protection against potential leaks, where the component to be unmounted and there is
// still a pending doConnect promise. This would lead the user to still be in the call even
// if the component is unmounted.
const abortHandlesBag = useRef(new Set<AbortHandle>());
// This is a cleanup function that will be called when the component is about to be unmounted.
// It will cancel all abortHandles in the bag
useEffect(() => {
const bag = abortHandlesBag.current;
return (): void => {
bag.forEach((handle) => {
handle.abort();
});
};
}, []);
// Id we are transitioning from a valid config to another valid one, we need
// to explicitly switch focus
useEffect(() => {
@@ -273,11 +323,14 @@ export function useECConnectionState(
// always capturing audio: it helps keep bluetooth headsets in the right mode and
// mobile browsers to know we're doing a call.
setIsInDoConnect(true);
const abortHandle = new AbortHandle();
abortHandlesBag.current.add(abortHandle);
doConnect(
livekitRoom!,
sfuConfig!,
initialAudioEnabled,
initialAudioOptions,
initialDeviceId,
abortHandle,
)
.catch((e) => {
if (e instanceof ElementCallError) {
@@ -286,14 +339,17 @@ export function useECConnectionState(
setError(new UnknownCallError(e));
} else logger.error("Failed to connect to SFU", e);
})
.finally(() => setIsInDoConnect(false));
.finally(() => {
abortHandlesBag.current.delete(abortHandle);
setIsInDoConnect(false);
});
}
currentSFUConfig.current = Object.assign({}, sfuConfig);
}, [
sfuConfig,
livekitRoom,
initialAudioOptions,
initialDeviceId,
initialAudioEnabled,
doFocusSwitch,
]);

67
src/livekit/useLiveKit.ts → src/livekit/useLivekit.ts
View File

@@ -9,20 +9,23 @@ import {
ConnectionState,
type E2EEManagerOptions,
ExternalE2EEKeyProvider,
LocalVideoTrack,
Room,
type RoomOptions,
Track,
} from "livekit-client";
import { useEffect, useMemo, useRef } from "react";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { map } from "rxjs";
import { defaultLiveKitOptions } from "./options";
import { type SFUConfig } from "./openIDSFU";
import { type MuteStates } from "../room/MuteStates";
import {
type MediaDevice,
type MediaDeviceHandle,
type MediaDevices,
useMediaDevices,
} from "./MediaDevicesContext";
@@ -33,27 +36,39 @@ import {
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "./TrackProcessorContext";
import { useInitial } from "../useInitial";
import { observeTrackReference$ } from "../state/MediaViewModel";
import { useUrlParams } from "../UrlParams";
interface UseLivekitResult {
livekitRoom?: Room;
connState: ECConnectionState;
}
export function useLiveKit(
export function useLivekit(
rtcSession: MatrixRTCSession,
muteStates: MuteStates,
sfuConfig: SFUConfig | undefined,
e2eeSystem: EncryptionSystem,
): UseLivekitResult {
const { controlledAudioDevices } = useUrlParams();
const e2eeOptions = useMemo((): E2EEManagerOptions | undefined => {
if (e2eeSystem.kind === E2eeType.NONE) return undefined;
if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
logger.info("Created MatrixKeyProvider (per participant)");
return {
keyProvider: new MatrixKeyProvider(),
worker: new E2EEWorker(),
};
} else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
logger.info("Created ExternalE2EEKeyProvider (shared key)");
return {
keyProvider: new ExternalE2EEKeyProvider(),
worker: new E2EEWorker(),
@@ -79,12 +94,15 @@ export function useLiveKit(
const devices = useMediaDevices();
const initialDevices = useRef<MediaDevices>(devices);
const { processor } = useTrackProcessor();
const initialProcessor = useInitial(() => processor);
const roomOptions = useMemo(
(): RoomOptions => ({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: initialDevices.current.videoInput.selectedId,
processor: initialProcessor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
@@ -95,7 +113,7 @@ export function useLiveKit(
},
e2ee: e2eeOptions,
}),
[e2eeOptions],
[e2eeOptions, initialProcessor],
);
// Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -113,6 +131,7 @@ export function useLiveKit(
// @livekit/components-react. JSON.stringify() is used in deps of a
// useEffect() with an argument that references itself, if E2EE is enabled
const room = useMemo(() => {
logger.info("[LivekitRooms] Create LiveKit room with options", roomOptions);
const r = new Room(roomOptions);
r.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
@@ -120,10 +139,27 @@ export function useLiveKit(
return r;
}, [roomOptions, e2eeSystem]);
// Sync the requested track processors with LiveKit
useTrackProcessorSync(
useObservableEagerState(
useObservable(
(room$) =>
observeTrackReference$(
room$.pipe(map(([room]) => room.localParticipant)),
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
[room],
),
),
);
const connectionState = useECConnectionState(
{
deviceId: initialDevices.current.audioInput.selectedId,
},
initialDevices.current.audioInput.selectedId,
initialMuteStates.current.audio.enabled,
room,
sfuConfig,
@@ -195,6 +231,7 @@ export function useLiveKit(
audioMuteUpdating.current = true;
trackPublication = await participant.setMicrophoneEnabled(
buttonEnabled.current.audio,
room.options.audioCaptureDefaults,
);
audioMuteUpdating.current = false;
break;
@@ -202,6 +239,7 @@ export function useLiveKit(
videoMuteUpdating.current = true;
trackPublication = await participant.setCameraEnabled(
buttonEnabled.current.video,
room.options.videoCaptureDefaults,
);
videoMuteUpdating.current = false;
break;
@@ -269,8 +307,15 @@ export function useLiveKit(
useEffect(() => {
// Sync the requested devices with LiveKit's devices
if (room !== undefined && connectionState === ConnectionState.Connected) {
const syncDevice = (kind: MediaDeviceKind, device: MediaDevice): void => {
if (
room !== undefined &&
connectionState === ConnectionState.Connected &&
!controlledAudioDevices
) {
const syncDevice = (
kind: MediaDeviceKind,
device: MediaDeviceHandle,
): void => {
const id = device.selectedId;
// Detect if we're trying to use chrome's default device, in which case
@@ -326,7 +371,7 @@ export function useLiveKit(
syncDevice("audiooutput", devices.audioOutput);
syncDevice("videoinput", devices.videoInput);
}
}, [room, devices, connectionState]);
}, [room, devices, connectionState, controlledAudioDevices]);
return {
connState: connectionState,

6
src/main.tsx
View File

@@ -9,12 +9,12 @@ Please see LICENSE in the repository root for full details.
// function gets set. It needs to be not in the same file as we use
// createClient, or the typescript transpiler gets confused about
// dependency references.
import "matrix-js-sdk/src/browser-index";
import "matrix-js-sdk/lib/browser-index";
import { StrictMode } from "react";
import { createRoot } from "react-dom/client";
import "./index.css";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import {
setLogExtension as setLKLogExtension,
setLogLevel as setLKLogLevel,
@@ -29,7 +29,7 @@ window.setLKLogLevel = setLKLogLevel;
initRageshake().catch((e) => {
logger.error("Failed to initialize rageshake", e);
});
setLKLogLevel("warn");
setLKLogLevel("info");
setLKLogExtension((level, msg, context) => {
// we pass a synthetic logger name of "livekit" to the rageshake to make it easier to read
global.mx_rage_logger.log(level, "livekit", msg, context);

5
src/mediapipe/imageSegmenter/README.md Normal file
View File

@@ -0,0 +1,5 @@
# Google AI Edge MediaPipe Selfie Segmentation
- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite
- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf

BIN
src/mediapipe/imageSegmenter/selfie_segmenter.tflite Normal file
View File

Binary file not shown.

6
src/otel/OTelCall.ts
View File

@@ -6,12 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
import { type Span } from "@opentelemetry/api";
import { type MatrixCall } from "matrix-js-sdk/src/matrix";
import { CallEvent } from "matrix-js-sdk/src/webrtc/call";
import { type MatrixCall } from "matrix-js-sdk";
import { CallEvent } from "matrix-js-sdk/lib/webrtc/call";
import {
type TransceiverStats,
type CallFeedStats,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { ObjectFlattener } from "./ObjectFlattener";
import { ElementCallOpenTelemetry } from "./otel";

2
src/otel/OTelCallAbstractMediaStreamSpan.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import opentelemetry, { type Span } from "@opentelemetry/api";
import { type TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport";
import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallMediaStreamTrackSpan } from "./OTelCallMediaStreamTrackSpan";

2
src/otel/OTelCallFeedMediaStreamSpan.ts
View File

@@ -9,7 +9,7 @@ import { type Span } from "@opentelemetry/api";
import {
type CallFeedStats,
type TrackStats,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";

2
src/otel/OTelCallMediaStreamTrackSpan.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport";
import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import opentelemetry, { type Span } from "@opentelemetry/api";
import { type ElementCallOpenTelemetry } from "./otel";

2
src/otel/OTelCallTransceiverMediaStreamSpan.ts
View File

@@ -9,7 +9,7 @@ import { type Span } from "@opentelemetry/api";
import {
type TrackStats,
type TransceiverStats,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";

10
src/otel/OTelGroupCallMembership.ts
View File

@@ -15,26 +15,26 @@ import {
type MatrixClient,
type MatrixEvent,
type RoomMember,
} from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type CallError,
type CallState,
type MatrixCall,
type VoipEvent,
} from "matrix-js-sdk/src/webrtc/call";
} from "matrix-js-sdk/lib/webrtc/call";
import {
type CallsByUserAndDevice,
type GroupCallError,
GroupCallEvent,
type GroupCallStatsReport,
} from "matrix-js-sdk/src/webrtc/groupCall";
} from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type ConnectionStatsReport,
type ByteSentStatsReport,
type SummaryStatsReport,
type CallFeedReport,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { ElementCallOpenTelemetry } from "./otel";
import { ObjectFlattener } from "./ObjectFlattener";

4
src/otel/ObjectFlattener.test.ts
View File

@@ -5,12 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type GroupCallStatsReport } from "matrix-js-sdk/src/webrtc/groupCall";
import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type AudioConcealment,
type ByteSentStatsReport,
type ConnectionStatsReport,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { describe, expect, it } from "vitest";
import { ObjectFlattener } from "../../src/otel/ObjectFlattener";

6
src/otel/ObjectFlattener.ts
View File

@@ -5,13 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Attributes } from "@opentelemetry/api";
import { type VoipEvent } from "matrix-js-sdk/src/webrtc/call";
import { type GroupCallStatsReport } from "matrix-js-sdk/src/webrtc/groupCall";
import { type VoipEvent } from "matrix-js-sdk/lib/webrtc/call";
import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type ByteSentStatsReport,
type ConnectionStatsReport,
type SummaryStatsReport,
} from "matrix-js-sdk/src/webrtc/stats/statsReport";
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
export class ObjectFlattener {
public static flattenReportObject(

37
src/otel/otel.ts
View File

@@ -5,13 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
import {
SimpleSpanProcessor,
type SpanProcessor,
} from "@opentelemetry/sdk-trace-base";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
import opentelemetry, { type Tracer } from "@opentelemetry/api";
import { Resource } from "@opentelemetry/resources";
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
import { logger } from "matrix-js-sdk/src/logger";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
import { logger } from "matrix-js-sdk/lib/logger";
import { PosthogSpanProcessor } from "../analytics/PosthogSpanProcessor";
import { Config } from "../config/Config";
@@ -59,34 +62,34 @@ export class ElementCallOpenTelemetry {
collectorUrl: string | undefined,
rageshakeUrl: string | undefined,
) {
// This is how we can make Jaeger show a reasonable service in the dropdown on the left.
const providerConfig = {
resource: new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME,
}),
};
this._provider = new WebTracerProvider(providerConfig);
const spanProcessors: SpanProcessor[] = [];
if (collectorUrl) {
logger.info("Enabling OTLP collector with URL " + collectorUrl);
this.otlpExporter = new OTLPTraceExporter({
url: collectorUrl,
});
this._provider.addSpanProcessor(
new SimpleSpanProcessor(this.otlpExporter),
);
spanProcessors.push(new SimpleSpanProcessor(this.otlpExporter));
} else {
logger.info("OTLP collector disabled");
}
if (rageshakeUrl) {
this.rageshakeProcessor = new RageshakeSpanProcessor();
this._provider.addSpanProcessor(this.rageshakeProcessor);
spanProcessors.push(this.rageshakeProcessor);
}
this._provider.addSpanProcessor(new PosthogSpanProcessor());
opentelemetry.trace.setGlobalTracerProvider(this._provider);
spanProcessors.push(new PosthogSpanProcessor());
this._provider = new WebTracerProvider({
resource: resourceFromAttributes({
// This is how we can make Jaeger show a reasonable service in the dropdown on the left.
[ATTR_SERVICE_NAME]: SERVICE_NAME,
}),
spanProcessors,
});
opentelemetry.trace.setGlobalTracerProvider(this._provider);
this._tracer = opentelemetry.trace.getTracer(
// This is not the serviceName shown in jaeger
"my-element-call-otl-tracer",

13
src/profile/useProfile.ts
View File

@@ -5,12 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixEvent } from "matrix-js-sdk/src/models/event";
import { type User, UserEvent } from "matrix-js-sdk/src/models/user";
import { type FileType } from "matrix-js-sdk/src/http-api";
import {
type MatrixEvent,
type User,
type MatrixClient,
UserEvent,
type FileType,
} from "matrix-js-sdk";
import { useState, useCallback, useEffect } from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
interface ProfileLoadState {
success: boolean;

4
src/reactions/ReactionsReader.test.tsx
View File

@@ -7,14 +7,14 @@ Please see LICENSE in the repository root for full details.
import { renderHook } from "@testing-library/react";
import { afterEach, test, vitest } from "vitest";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import {
RoomEvent as MatrixRoomEvent,
MatrixEvent,
type IRoomTimelineData,
EventType,
MatrixEventEvent,
} from "matrix-js-sdk/src/matrix";
} from "matrix-js-sdk";
import { ReactionsReader, REACTION_ACTIVE_TIME_MS } from "./ReactionsReader";
import {

10
src/reactions/ReactionsReader.ts
View File

@@ -9,15 +9,15 @@ import {
type CallMembership,
MatrixRTCSessionEvent,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/matrix";
import { type ReactionEventContent } from "matrix-js-sdk/src/types";
} from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixEvent, MatrixEventEvent } from "matrix-js-sdk";
import { type ReactionEventContent } from "matrix-js-sdk/lib/types";
import {
RelationType,
EventType,
RoomEvent as MatrixRoomEvent,
} from "matrix-js-sdk/src/matrix";
} from "matrix-js-sdk";
import { BehaviorSubject, delay, type Subscription } from "rxjs";
import {

2
src/reactions/index.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RelationType } from "matrix-js-sdk/src/types";
import { type RelationType } from "matrix-js-sdk";
import catSoundOgg from "../sound/reactions/cat.ogg?url";
import catSoundMp3 from "../sound/reactions/cat.mp3?url";

6
src/reactions/useReactionsSender.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { EventType, RelationType } from "matrix-js-sdk/src/matrix";
import { EventType, RelationType } from "matrix-js-sdk";
import {
createContext,
useContext,
@@ -14,8 +14,8 @@ import {
useMemo,
type JSX,
} from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { useObservableEagerState } from "observable-hooks";
import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";

2
src/room/AppSelectionModal.tsx
View File

@@ -15,7 +15,7 @@ import {
import { useTranslation } from "react-i18next";
import { Button, Text } from "@vector-im/compound-web";
import { PopOutIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Modal } from "../Modal";
import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";

4
src/room/CallEndedView.tsx
View File

@@ -6,11 +6,11 @@ Please see LICENSE in the repository root for full details.
*/
import { type FC, type FormEventHandler, useCallback, useState } from "react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import { Trans, useTranslation } from "react-i18next";
import { Button, Heading, Text } from "@vector-im/compound-web";
import { useNavigate } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import styles from "./CallEndedView.module.css";
import feedbackStyle from "../input/FeedbackInput.module.css";

2
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -16,7 +16,7 @@ import {
afterEach,
} from "vitest";
import { act } from "react";
import { type CallMembership } from "matrix-js-sdk/src/matrixrtc";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { mockRtcMembership } from "../utils/test";
import {

3
src/room/CallEventAudioRenderer.tsx
View File

@@ -47,12 +47,15 @@ export const callEventAudioSounds = prefetchSounds({
export function CallEventAudioRenderer({
vm,
muted,
}: {
vm: CallViewModel;
muted?: boolean;
}): ReactNode {
const audioEngineCtx = useAudioContext({
sounds: callEventAudioSounds,
latencyHint: "interactive",
muted,
});
const audioEngineRef = useLatest(audioEngineCtx);

41
src/room/GroupCallView.test.tsx
View File

@@ -14,13 +14,12 @@ import {
vi,
} from "vitest";
import { render, waitFor, screen } from "@testing-library/react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { of } from "rxjs";
import { JoinRule, type RoomState } from "matrix-js-sdk/src/matrix";
import { BrowserRouter } from "react-router-dom";
import userEvent from "@testing-library/user-event";
import { type RelationsContainer } from "matrix-js-sdk/src/models/relations-container";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
import { useState } from "react";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -39,6 +38,7 @@ import { GroupCallView } from "./GroupCallView";
import { type WidgetHelpers } from "../widget";
import { LazyEventEmitter } from "../LazyEventEmitter";
import { MatrixRTCFocusMissingError } from "../utils/errors";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
vi.mock("../soundUtils");
vi.mock("../useAudioContext");
@@ -47,6 +47,13 @@ vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
}));
vi.hoisted(
() =>
(global.ImageData = class MockImageData {
public data: number[] = [];
} as unknown as typeof ImageData),
);
const enterRTCSession = vi.hoisted(() => vi.fn(async () => Promise.resolve()));
const leaveRTCSession = vi.hoisted(() =>
vi.fn(
@@ -138,18 +145,20 @@ function createGroupCallView(
const { getByText } = render(
<BrowserRouter>
<TooltipProvider>
<GroupCallView
client={client}
isPasswordlessUser={false}
confineToRoom={false}
preload={false}
skipLobby={false}
hideHeader={true}
rtcSession={rtcSession as unknown as MatrixRTCSession}
isJoined={joined}
muteStates={muteState}
widget={widget}
/>
<ProcessorProvider>
<GroupCallView
client={client}
isPasswordlessUser={false}
confineToRoom={false}
preload={false}
skipLobby={false}
hideHeader={true}
rtcSession={rtcSession as unknown as MatrixRTCSession}
isJoined={joined}
muteStates={muteState}
widget={widget}
/>
</ProcessorProvider>
</TooltipProvider>
</BrowserRouter>,
);

73
src/room/GroupCallView.tsx
View File

@@ -13,18 +13,18 @@ import {
useMemo,
useState,
} from "react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient, JoinRule, type Room } from "matrix-js-sdk";
import {
Room as LivekitRoom,
isE2EESupported as isE2EESupportedBrowser,
} from "livekit-client";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import {
MatrixRTCSessionEvent,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { JoinRule, type Room } from "matrix-js-sdk/src/matrix";
} from "matrix-js-sdk/lib/matrixrtc";
import { useNavigate } from "react-router-dom";
import { useObservableEagerState } from "observable-hooks";
import type { IWidgetApiRequest } from "matrix-widget-api";
import {
@@ -63,10 +63,12 @@ import {
} from "../utils/errors.ts";
import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx";
import {
useNewMembershipManagerSetting as useNewMembershipManagerSetting,
useNewMembershipManager as useNewMembershipManagerSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
useSetting,
} from "../settings/settings";
import { useTypedEventEmitter } from "../useEvents";
import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
declare global {
interface Window {
@@ -103,12 +105,14 @@ export const GroupCallView: FC<Props> = ({
const [externalError, setExternalError] = useState<ElementCallError | null>(
null,
);
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const muteAllAudio = useObservableEagerState(muteAllAudio$);
const leaveSoundContext = useLatest(
useAudioContext({
sounds: callEventAudioSounds,
latencyHint: "interactive",
muted: muteAllAudio,
}),
);
// This should use `useEffectEvent` (only available in experimental versions)
@@ -118,6 +122,13 @@ export const GroupCallView: FC<Props> = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
useEffect(() => {
logger.info("[Lifecycle] GroupCallView Component mounted");
return (): void => {
logger.info("[Lifecycle] GroupCallView Component unmounted");
};
}, []);
useEffect(() => {
window.rtcSession = rtcSession;
return (): void => {
@@ -152,6 +163,9 @@ export const GroupCallView: FC<Props> = ({
const { perParticipantE2EE, returnToLobby } = useUrlParams();
const e2eeSystem = useRoomEncryptionSystem(room.roomId);
const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting);
const [useExperimentalToDeviceTransport] = useSetting(
useExperimentalToDeviceTransportSetting,
);
usePageTitle(roomName);
@@ -179,16 +193,13 @@ export const GroupCallView: FC<Props> = ({
const latestMuteStates = useLatest(muteStates);
const enterRTCSessionOrError = useCallback(
async (
rtcSession: MatrixRTCSession,
perParticipantE2EE: boolean,
newMembershipManager: boolean,
): Promise<void> => {
async (rtcSession: MatrixRTCSession): Promise<void> => {
try {
await enterRTCSession(
rtcSession,
perParticipantE2EE,
newMembershipManager,
useNewMembershipManager,
useExperimentalToDeviceTransport,
);
} catch (e) {
if (e instanceof ElementCallError) {
@@ -202,7 +213,11 @@ export const GroupCallView: FC<Props> = ({
}
}
},
[setExternalError],
[
perParticipantE2EE,
useExperimentalToDeviceTransport,
useNewMembershipManager,
],
);
useEffect(() => {
@@ -254,11 +269,7 @@ export const GroupCallView: FC<Props> = ({
await defaultDeviceSetup(
ev.detail.data as unknown as JoinCallData,
);
await enterRTCSessionOrError(
rtcSession,
perParticipantE2EE,
useNewMembershipManager,
);
await enterRTCSessionOrError(rtcSession);
widget.api.transport.reply(ev.detail, {});
})().catch((e) => {
logger.error("Error joining RTC session", e);
@@ -271,21 +282,13 @@ export const GroupCallView: FC<Props> = ({
} else {
// No lobby and no preload: we enter the rtc session right away
(async (): Promise<void> => {
await enterRTCSessionOrError(
rtcSession,
perParticipantE2EE,
useNewMembershipManager,
);
await enterRTCSessionOrError(rtcSession);
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
}
} else {
void enterRTCSessionOrError(
rtcSession,
perParticipantE2EE,
useNewMembershipManager,
);
void enterRTCSessionOrError(rtcSession);
}
}
}, [
@@ -408,13 +411,7 @@ export const GroupCallView: FC<Props> = ({
client={client}
matrixInfo={matrixInfo}
muteStates={muteStates}
onEnter={() =>
void enterRTCSessionOrError(
rtcSession,
perParticipantE2EE,
useNewMembershipManager,
)
}
onEnter={() => void enterRTCSessionOrError(rtcSession)}
confineToRoom={confineToRoom}
hideHeader={hideHeader}
participantCount={participantCount}
@@ -492,11 +489,7 @@ export const GroupCallView: FC<Props> = ({
recoveryActionHandler={(action) => {
if (action == "reconnect") {
setLeft(false);
enterRTCSessionOrError(
rtcSession,
perParticipantE2EE,
useNewMembershipManager,
).catch((e) => {
enterRTCSessionOrError(rtcSession).catch((e) => {
logger.error("Error re-entering RTC session", e);
});
}

265
src/room/InCallView.test.tsx Normal file
View File

@@ -0,0 +1,265 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
describe,
expect,
it,
type MockedFunction,
vi,
} from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
import { ConnectionState, type LocalParticipant } from "livekit-client";
import { of } from "rxjs";
import { BrowserRouter } from "react-router-dom";
import { TooltipProvider } from "@vector-im/compound-web";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type MuteStates } from "./MuteStates";
import { InCallView } from "./InCallView";
import {
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockRemoteParticipant,
mockRtcMembership,
type MockRTCSession,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { alice, local } from "../utils/test-fixtures";
import {
developerMode as developerModeSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
} from "../settings/settings";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
// vi.hoisted(() => {
// localStorage = {} as unknown as Storage;
// });
vi.hoisted(
() =>
(global.ImageData = class MockImageData {
public data: number[] = [];
} as unknown as typeof ImageData),
);
vi.mock("../soundUtils");
vi.mock("../useAudioContext");
vi.mock("../tile/GridTile");
vi.mock("../tile/SpotlightTile");
vi.mock("@livekit/components-react");
vi.mock("../e2ee/sharedKeyManagement");
vi.mock("../livekit/MatrixAudioRenderer");
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
}));
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const localParticipant = mockLocalParticipant({
identity: "@local:example.org:AAAAAA",
});
const remoteParticipant = mockRemoteParticipant({
identity: "@alice:example.org:AAAAAA",
});
const carol = mockMatrixRoomMember(localRtcMember);
const roomMembers = new Map([carol].map((p) => [p.userId, p]));
const roomId = "!foo:bar";
let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
beforeEach(() => {
vi.clearAllMocks();
// MatrixAudioRenderer is tested separately.
(
MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
).mockImplementation((_props) => {
return <div>mocked: MatrixAudioRenderer</div>;
});
(
useLocalParticipant as MockedFunction<typeof useLocalParticipant>
).mockImplementation(
() =>
({
isScreenShareEnabled: false,
localParticipant: localRtcMember as unknown as LocalParticipant,
}) as unknown as ReturnType<typeof useLocalParticipant>,
);
useRoomEncryptionSystemMock =
useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });
});
function createInCallView(): RenderResult & {
rtcSession: MockRTCSession;
} {
const client = {
getUser: () => null,
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;
const room = mockMatrixRoom({
relations: {
getChildEventsForEvent: () =>
vi.mocked({
getRelations: () => [],
}),
} as unknown as RelationsContainer,
client,
roomId,
getMember: (userId) => roomMembers.get(userId) ?? null,
getMxcAvatarUrl: () => null,
hasEncryptionStateEvent: vi.fn().mockReturnValue(true),
getCanonicalAlias: () => null,
currentState: {
getJoinRule: () => JoinRule.Invite,
} as Partial<RoomState> as RoomState,
});
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
const livekitRoom = mockLivekitRoom(
{
localParticipant,
},
{
remoteParticipants$: of([remoteParticipant]),
},
);
const { vm, rtcSession } = getBasicCallViewModelEnvironment([local, alice]);
rtcSession.joined = true;
const renderResult = render(
<BrowserRouter>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<TooltipProvider>
<RoomContext.Provider value={livekitRoom}>
<InCallView
client={client}
hideHeader={true}
rtcSession={rtcSession as unknown as MatrixRTCSession}
muteStates={muteState}
vm={vm}
matrixInfo={{
userId: "",
displayName: "",
avatarUrl: "",
roomId: "",
roomName: "",
roomAlias: null,
roomAvatar: null,
e2eeSystem: {
kind: E2eeType.NONE,
},
}}
livekitRoom={livekitRoom}
participantCount={0}
onLeave={function (): void {
throw new Error("Function not implemented.");
}}
connState={ConnectionState.Connected}
onShareClick={null}
/>
</RoomContext.Provider>
</TooltipProvider>
</ReactionsSenderProvider>
</BrowserRouter>,
);
return {
...renderResult,
rtcSession,
};
}
describe("InCallView", () => {
describe("rendering", () => {
it("renders", () => {
const { container } = createInCallView();
expect(container).toMatchSnapshot();
});
});
describe("toDevice label", () => {
it("is shown if setting activated and room encrypted", () => {
useRoomEncryptionSystemMock.mockReturnValue({
kind: E2eeType.PER_PARTICIPANT,
});
useExperimentalToDeviceTransportSetting.setValue(true);
developerModeSetting.setValue(true);
const { getByText } = createInCallView();
expect(getByText("using to Device key transport")).toBeInTheDocument();
});
it("is not shown in unenecrypted room", () => {
useRoomEncryptionSystemMock.mockReturnValue({
kind: E2eeType.NONE,
});
useExperimentalToDeviceTransportSetting.setValue(true);
developerModeSetting.setValue(true);
const { queryByText } = createInCallView();
expect(
queryByText("using to Device key transport"),
).not.toBeInTheDocument();
});
it("is hidden once fallback was triggered", async () => {
useRoomEncryptionSystemMock.mockReturnValue({
kind: E2eeType.PER_PARTICIPANT,
});
useExperimentalToDeviceTransportSetting.setValue(true);
developerModeSetting.setValue(true);
const { rtcSession, queryByText } = createInCallView();
expect(queryByText("using to Device key transport")).toBeInTheDocument();
expect(rtcSession).toBeDefined();
await act(() =>
rtcSession.emit(RoomAndToDeviceEvents.EnabledTransportsChanged, {
toDevice: true,
room: true,
}),
);
expect(
queryByText("using to Device key transport"),
).not.toBeInTheDocument();
});
it("is not shown if setting is disabled", () => {
useExperimentalToDeviceTransportSetting.setValue(false);
developerModeSetting.setValue(true);
useRoomEncryptionSystemMock.mockReturnValue({
kind: E2eeType.PER_PARTICIPANT,
});
const { queryByText } = createInCallView();
expect(
queryByText("using to Device key transport"),
).not.toBeInTheDocument();
});
it("is not shown if developer mode is disabled", () => {
useExperimentalToDeviceTransportSetting.setValue(true);
developerModeSetting.setValue(false);
useRoomEncryptionSystemMock.mockReturnValue({
kind: E2eeType.PER_PARTICIPANT,
});
const { queryByText } = createInCallView();
expect(
queryByText("using to Device key transport"),
).not.toBeInTheDocument();
});
});
});

98
src/room/InCallView.tsx
View File

@@ -5,13 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
RoomAudioRenderer,
RoomContext,
useLocalParticipant,
} from "@livekit/components-react";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { Text } from "@vector-im/compound-web";
import { ConnectionState, type Room } from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import {
type FC,
type PointerEvent,
@@ -26,11 +23,12 @@ import {
type JSX,
} from "react";
import useMeasure from "react-use-measure";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -54,7 +52,7 @@ import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { useLiveKit } from "../livekit/useLiveKit";
import { useLivekit } from "../livekit/useLivekit.ts";
import { useWakeLock } from "../useWakeLock";
import { useMergedRefs } from "../useMergedRefs";
import { type MuteStates } from "./MuteStates";
@@ -71,7 +69,10 @@ import {
import { Grid, type TileProps } from "../grid/Grid";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
useRoomEncryptionSystem,
type EncryptionSystem,
} from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import {
@@ -94,10 +95,15 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import {
debugTileLayout as debugTileLayoutSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
developerMode as developerModeSetting,
useSetting,
} from "../settings/settings";
import { ReactionsReader } from "../reactions/ReactionsReader";
import { ConnectionLostError } from "../utils/errors.ts";
import { useTypedEventEmitter } from "../useEvents.ts";
import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -110,7 +116,7 @@ export interface ActiveCallProps
export const ActiveCall: FC<ActiveCallProps> = (props) => {
const sfuConfig = useOpenIDSFU(props.client, props.rtcSession);
const { livekitRoom, connState } = useLiveKit(
const { livekitRoom, connState } = useLivekit(
props.rtcSession,
props.muteStates,
sfuConfig,
@@ -123,10 +129,23 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const [vm, setVm] = useState<CallViewModel | null>(null);
useEffect(() => {
logger.info(
`[Lifecycle] InCallView Component mounted, livekitroom state ${livekitRoom?.state}`,
);
return (): void => {
livekitRoom?.disconnect().catch((e) => {
logger.error("Failed to disconnect from livekit room", e);
});
logger.info(
`[Lifecycle] InCallView Component unmounted, livekitroom state ${livekitRoom?.state}`,
);
livekitRoom
?.disconnect()
.then(() => {
logger.info(
`[Lifecycle] Disconnected from livekite room, state:${livekitRoom?.state}`,
);
})
.catch((e) => {
logger.error("[Lifecycle] Failed to disconnect from livekit room", e);
});
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
@@ -216,6 +235,36 @@ export const InCallView: FC<InCallViewProps> = ({
room: livekitRoom,
});
const muteAllAudio = useObservableEagerState(muteAllAudio$);
// This seems like it might be enough logic to use move it into the call view model?
const [didFallbackToRoomKey, setDidFallbackToRoomKey] = useState(false);
useTypedEventEmitter(
rtcSession,
RoomAndToDeviceEvents.EnabledTransportsChanged,
(enabled) => setDidFallbackToRoomKey(enabled.room),
);
const [developerMode] = useSetting(developerModeSetting);
const [useExperimentalToDeviceTransport] = useSetting(
useExperimentalToDeviceTransportSetting,
);
const encryptionSystem = useRoomEncryptionSystem(rtcSession.room.roomId);
const showToDeviceEncryption = useMemo(
() =>
developerMode &&
useExperimentalToDeviceTransport &&
encryptionSystem.kind === E2eeType.PER_PARTICIPANT &&
!didFallbackToRoomKey,
[
developerMode,
useExperimentalToDeviceTransport,
encryptionSystem.kind,
didFallbackToRoomKey,
],
);
const toggleMicrophone = useCallback(
() => muteStates.audio.setEnabled?.((e) => !e),
[muteStates],
@@ -662,10 +711,25 @@ export const InCallView: FC<InCallViewProps> = ({
</RightNav>
</Header>
))}
<RoomAudioRenderer />
{
// TODO: remove this once we remove the developer flag gets removed and we have shipped to
// device transport as the default.
showToDeviceEncryption && (
<Text
style={{ height: 0, zIndex: 1, alignSelf: "center", margin: 0 }}
size="sm"
>
using to Device key transport
</Text>
)
}
<MatrixAudioRenderer
members={rtcSession.memberships}
muted={muteAllAudio}
/>
{renderContent()}
<CallEventAudioRenderer vm={vm} />
<ReactionsAudioRenderer vm={vm} />
<CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
<ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
<ReactionsOverlay vm={vm} />
{footer}
{layout.type !== "pip" && (

2
src/room/InviteModal.test.tsx
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { render, screen } from "@testing-library/react";
import { expect, test, vi } from "vitest";
import { type Room } from "matrix-js-sdk/src/matrix";
import { type Room } from "matrix-js-sdk";
import { axe } from "vitest-axe";
import { BrowserRouter } from "react-router-dom";
import userEvent from "@testing-library/user-event";

2
src/room/InviteModal.tsx
View File

@@ -13,7 +13,7 @@ import {
useState,
} from "react";
import { useTranslation } from "react-i18next";
import { type Room } from "matrix-js-sdk/src/matrix";
import { type Room } from "matrix-js-sdk";
import { Button, Text } from "@vector-im/compound-web";
import {
LinkIcon,

49
src/room/LobbyView.tsx
View File

@@ -5,14 +5,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, useCallback, useMemo, useState, type JSX } from "react";
import {
type FC,
useCallback,
useMemo,
useState,
type JSX,
useEffect,
} from "react";
import { useTranslation } from "react-i18next";
import { type MatrixClient } from "matrix-js-sdk/src/matrix";
import { type MatrixClient } from "matrix-js-sdk";
import { Button } from "@vector-im/compound-web";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { usePreviewTracks } from "@livekit/components-react";
import { type LocalVideoTrack, Track } from "livekit-client";
import {
type CreateLocalTracksOptions,
type LocalVideoTrack,
Track,
} from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import { useNavigate } from "react-router-dom";
@@ -36,7 +47,11 @@ import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "../livekit/TrackProcessorContext";
import { usePageTitle } from "../usePageTitle";
interface Props {
@@ -64,6 +79,13 @@ export const LobbyView: FC<Props> = ({
onShareClick,
waitingForInvite,
}) => {
useEffect(() => {
logger.info("[Lifecycle] GroupCallView Component mounted");
return (): void => {
logger.info("[Lifecycle] GroupCallView Component unmounted");
};
}, []);
const { t } = useTranslation();
usePageTitle(matrixInfo.roomName);
@@ -112,7 +134,10 @@ export const LobbyView: FC<Props> = ({
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);
const localTrackOptions = useMemo(
const { processor } = useTrackProcessor();
const initialProcessor = useInitial(() => processor);
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
() => ({
// The only reason we request audio here is to get the audio permission
// request over with at the same time. But changing the audio settings
@@ -123,12 +148,14 @@ export const LobbyView: FC<Props> = ({
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
processor: initialProcessor,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
devices.videoInput.selectedId,
initialProcessor,
],
);
@@ -149,8 +176,8 @@ export const LobbyView: FC<Props> = ({
null) as LocalVideoTrack | null,
[tracks],
);
const switchCamera = useSwitchCamera(
useTrackProcessorSync(videoTrack);
const showSwitchCamera = useShowSwitchCamera(
useObservable(
(inputs$) => inputs$.pipe(map(([video]) => video)),
[videoTrack],
@@ -212,7 +239,9 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
{showSwitchCamera && (
<SwitchCameraButton onClick={showSwitchCamera} />
)}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>

5
src/room/MuteStates.test.tsx
View File

@@ -14,7 +14,7 @@ import userEvent from "@testing-library/user-event";
import { useMuteStates } from "./MuteStates";
import {
type DeviceLabel,
type MediaDevice,
type MediaDeviceHandle,
type MediaDevices,
MediaDevicesContext,
} from "../livekit/MediaDevicesContext";
@@ -73,12 +73,13 @@ const mockCamera: MediaDeviceInfo = {
},
};
function mockDevices(available: Map<string, DeviceLabel>): MediaDevice {
function mockDevices(available: Map<string, DeviceLabel>): MediaDeviceHandle {
return {
available,
selectedId: "",
selectedGroupId: "",
select: (): void => {},
useAsEarpiece: false,
};
}

6
src/room/MuteStates.ts
View File

@@ -13,10 +13,10 @@ import {
useMemo,
} from "react";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MediaDevice,
type MediaDeviceHandle,
useMediaDevices,
} from "../livekit/MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
@@ -53,7 +53,7 @@ export interface MuteStates {
}
function useMuteState(
device: MediaDevice,
device: MediaDeviceHandle,
enabledByDefault: () => boolean,
): MuteState {
const [enabled, setEnabled] = useReactiveState<boolean | undefined>(

14
src/room/ReactionAudioRenderer.test.tsx
View File

@@ -21,8 +21,8 @@ import { act, type ReactNode } from "react";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import {
playReactionsSound,
soundEffectVolumeSetting,
playReactionsSound as playReactionsSoundSetting,
soundEffectVolume as soundEffectVolumeSetting,
} from "../settings/settings";
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
@@ -50,7 +50,7 @@ vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();
playReactionsSound.setValue(playReactionsSound.defaultValue);
playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
});
@@ -74,7 +74,7 @@ beforeEach(() => {
test("preloads all audio elements", () => {
const { vm } = getBasicCallViewModelEnvironment([local, alice]);
playReactionsSound.setValue(true);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
@@ -84,7 +84,7 @@ test("will play an audio sound when there is a reaction", () => {
local,
alice,
]);
playReactionsSound.setValue(true);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
@@ -110,7 +110,7 @@ test("will play the generic audio sound when there is soundless reaction", () =>
local,
alice,
]);
playReactionsSound.setValue(true);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
@@ -136,7 +136,7 @@ test("will play multiple audio sounds when there are multiple different reaction
local,
alice,
]);
playReactionsSound.setValue(true);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect

3
src/room/ReactionAudioRenderer.tsx
View File

@@ -24,8 +24,10 @@ const soundMap = Object.fromEntries([
export function ReactionsAudioRenderer({
vm,
muted,
}: {
vm: CallViewModel;
muted?: boolean;
}): ReactNode {
const [shouldPlay] = useSetting(playReactionsSound);
const [soundCache, setSoundCache] = useState<ReturnType<
@@ -34,6 +36,7 @@ export function ReactionsAudioRenderer({
const audioEngineCtx = useAudioContext({
sounds: soundCache,
latencyHint: "interactive",
muted,
});
const audioEngineRef = useLatest(audioEngineCtx);

8
src/room/RoomAuthView.tsx
View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { type FC, useCallback, useState } from "react";
import { useLocation } from "react-router-dom";
import { Trans, useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Button, Heading, Text } from "@vector-im/compound-web";
import styles from "./RoomAuthView.module.css";
@@ -80,10 +80,10 @@ export const RoomAuthView: FC = () => {
/>
</FieldRow>
<Text size="sm">
<Trans i18nKey="room_auth_view_eula_caption">
<Trans i18nKey="room_auth_view_ssla_caption">
By clicking "Join call now", you agree to our{" "}
<ExternalLink href={Config.get().eula}>
End User Licensing Agreement (EULA)
<ExternalLink href={Config.get().ssla}>
Software and Services License Agreement (SSLA)
</ExternalLink>
</Trans>
</Text>

4
src/room/RoomPage.tsx
View File

@@ -13,13 +13,13 @@ import {
useRef,
type JSX,
} from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixError } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { Trans, useTranslation } from "react-i18next";
import {
CheckIcon,
UnknownSolidIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { type MatrixError } from "matrix-js-sdk/src/http-api";
import { useClientLegacy } from "../ClientContext";
import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView";

181
src/room/__snapshots__/InCallView.test.tsx.snap Normal file
View File

@@ -0,0 +1,181 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`InCallView > rendering > renders 1`] = `
<div>
<div
class="inRoom"
>
<div
class="header filler"
/>
<div>
mocked: MatrixAudioRenderer
</div>
<div
class="scrollingGrid grid"
>
<div
class="layer"
>
<div
class="container slot"
data-id="1"
>
<div
class="slot local slot"
data-block-alignment="start"
data-id="0"
data-inline-alignment="end"
/>
</div>
</div>
</div>
<div
class="fixedGrid grid"
>
<div />
</div>
<div
class="container"
/>
<div
class="footer"
>
<div
class="buttons"
>
<button
aria-disabled="false"
aria-labelledby=":r0:"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
data-kind="primary"
data-size="lg"
data-testid="incall_mute"
role="button"
tabindex="0"
>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M8 8v-.006l6.831 6.832-.002.002 1.414 1.415.003-.003 1.414 1.414-.003.003L20.5 20.5a1 1 0 0 1-1.414 1.414l-3.022-3.022A7.95 7.95 0 0 1 13 19.938V21a1 1 0 0 1-2 0v-1.062A8 8 0 0 1 4 12a1 1 0 1 1 2 0 6 6 0 0 0 8.587 5.415l-1.55-1.55A4.005 4.005 0 0 1 8 12v-1.172L2.086 4.914A1 1 0 0 1 3.5 3.5zm9.417 6.583 1.478 1.477A7.96 7.96 0 0 0 20 12a1 1 0 0 0-2 0c0 .925-.21 1.8-.583 2.583M8.073 5.238l7.793 7.793q.132-.495.134-1.031V6a4 4 0 0 0-7.927-.762"
/>
</svg>
</button>
<button
aria-disabled="false"
aria-labelledby=":r5:"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
data-kind="primary"
data-size="lg"
data-testid="incall_videomute"
role="button"
tabindex="0"
>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M2.747 2.753 4.35 4.355l.007-.003L18 17.994v.012l3.247 3.247a1 1 0 0 1-1.414 1.414l-2.898-2.898A2 2 0 0 1 16 20H6a4 4 0 0 1-4-4V8c0-.892.292-1.715.785-2.38L1.333 4.166a1 1 0 0 1 1.414-1.414M18 15.166 6.834 4H16a2 2 0 0 1 2 2v4.286l3.35-2.871a1 1 0 0 1 1.65.76v7.65a1 1 0 0 1-1.65.76L18 13.715z"
/>
</svg>
</button>
<button
aria-labelledby=":ra:"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
data-kind="secondary"
data-size="lg"
role="button"
tabindex="0"
>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M12.731 2C13.432 2 14 2.568 14 3.269c0 .578.396 1.074.935 1.286q.128.052.253.106c.531.23 1.162.16 1.572-.25a1.27 1.27 0 0 1 1.794 0l1.034 1.035a1.27 1.27 0 0 1 0 1.794c-.41.41-.48 1.04-.248 1.572l.105.253c.212.539.708.935 1.286.935.701 0 1.269.568 1.269 1.269v1.462c0 .701-.568 1.269-1.269 1.269-.578 0-1.074.396-1.287.935q-.05.128-.104.253c-.232.531-.161 1.162.248 1.572a1.27 1.27 0 0 1 0 1.794l-1.034 1.034a1.27 1.27 0 0 1-1.794 0c-.41-.41-1.04-.48-1.572-.248a8 8 0 0 1-.253.105c-.539.212-.935.708-.935 1.286 0 .701-.568 1.269-1.269 1.269H11.27c-.702 0-1.27-.568-1.27-1.269 0-.578-.396-1.074-.935-1.287a8 8 0 0 1-.253-.104c-.531-.232-1.162-.161-1.572.248a1.27 1.27 0 0 1-1.794 0l-1.034-1.034a1.27 1.27 0 0 1 0-1.794c.41-.41.48-1.04.249-1.572a8 8 0 0 1-.106-.253C4.343 14.396 3.847 14 3.27 14 2.568 14 2 13.432 2 12.731V11.27c0-.702.568-1.27 1.269-1.27.578 0 1.074-.396 1.286-.935q.052-.128.106-.253c.23-.531.16-1.162-.25-1.572a1.27 1.27 0 0 1 0-1.794l1.035-1.034a1.27 1.27 0 0 1 1.794 0c.41.41 1.04.48 1.572.249a8 8 0 0 1 .253-.106c.539-.212.935-.708.935-1.286C10 2.568 10.568 2 11.269 2zM12 16a4 4 0 1 0 0-8 4 4 0 0 0 0 8"
/>
</svg>
</button>
<button
aria-labelledby=":rf:"
class="_button_vczzf_8 endCall _has-icon_vczzf_57 _icon-only_vczzf_50 _destructive_vczzf_107"
data-kind="primary"
data-size="lg"
data-testid="incall_leave"
role="button"
tabindex="0"
>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m2.765 16.02-2.47-2.416A1.02 1.02 0 0 1 0 12.852q0-.456.295-.751a15.6 15.6 0 0 1 5.316-3.786A15.9 15.9 0 0 1 12 7q3.355 0 6.39 1.329a16 16 0 0 1 5.315 3.772q.295.294.295.751t-.295.752l-2.47 2.416a1.047 1.047 0 0 1-1.396.108l-3.114-2.363a1.1 1.1 0 0 1-.322-.376 1.1 1.1 0 0 1-.108-.483v-2.27a13.6 13.6 0 0 0-2.12-.524C13.459 9.996 12 9.937 12 9.937s-1.459.059-2.174.175q-1.074.174-2.121.523v2.271q0 .268-.108.483a1.1 1.1 0 0 1-.322.376l-3.114 2.363a1.047 1.047 0 0 1-1.396-.107"
/>
</svg>
</button>
</div>
<div
class="toggle layout"
>
<input
aria-labelledby=":rk:"
name="layout"
type="radio"
value="spotlight"
/>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M5 5h14v8h-5a1 1 0 0 0-1 1v5H5zm10 14v-4h4v4zM5 21h14a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2"
/>
</svg>
<input
aria-labelledby=":rp:"
checked=""
name="layout"
type="radio"
value="grid"
/>
<svg
aria-hidden="true"
fill="currentColor"
height="24"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M4 11a.97.97 0 0 1-.712-.287A.97.97 0 0 1 3 10V4q0-.424.288-.712A.97.97 0 0 1 4 3h6q.424 0 .713.288Q11 3.575 11 4v6q0 .424-.287.713A.97.97 0 0 1 10 11zm5-2V5H5v4zm5 12a.97.97 0 0 1-.713-.288A.97.97 0 0 1 13 20v-6q0-.424.287-.713A.97.97 0 0 1 14 13h6q.424 0 .712.287.288.288.288.713v6q0 .424-.288.712A.97.97 0 0 1 20 21zm5-2v-4h-4v4zM4 21a.97.97 0 0 1-.712-.288A.97.97 0 0 1 3 20v-6q0-.424.288-.713A.97.97 0 0 1 4 13h6q.424 0 .713.287.287.288.287.713v6q0 .424-.287.712A.97.97 0 0 1 10 21zm5-2v-4H5v4zm5-8a.97.97 0 0 1-.713-.287A.97.97 0 0 1 13 10V4q0-.424.287-.712A.97.97 0 0 1 14 3h6q.424 0 .712.288Q21 3.575 21 4v6q0 .424-.288.713A.97.97 0 0 1 20 11zm5-2V5h-4v4z"
/>
</svg>
</div>
</div>
</div>
</div>
`;

2
src/room/checkForParallelCalls.test.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { vi, type Mocked, test, expect } from "vitest";
import { type RoomState } from "matrix-js-sdk/src/models/room-state";
import { type RoomState } from "matrix-js-sdk";
import { PosthogAnalytics } from "../../src/analytics/PosthogAnalytics";
import { checkForParallelCalls } from "../../src/room/checkForParallelCalls";

3
src/room/checkForParallelCalls.ts
View File

@@ -5,8 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { EventType } from "matrix-js-sdk/src/@types/event";
import { type RoomState } from "matrix-js-sdk/src/models/room-state";
import { EventType, type RoomState } from "matrix-js-sdk";
import { PosthogAnalytics } from "../analytics/PosthogAnalytics";

11
src/room/useActiveFocus.ts
View File

@@ -8,14 +8,11 @@ Please see LICENSE in the repository root for full details.
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
} from "matrix-js-sdk/lib/matrixrtc";
import { useCallback, useEffect, useState } from "react";
import { deepCompare } from "matrix-js-sdk/src/utils";
import { logger } from "matrix-js-sdk/src/logger";
import {
type LivekitFocus,
isLivekitFocus,
} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LivekitFocus, isLivekitFocus } from "matrix-js-sdk/lib/matrixrtc";
/**
* Gets the currently active (livekit) focus for a MatrixRTC session

3
src/room/useJoinRule.ts
View File

@@ -6,9 +6,8 @@ Please see LICENSE in the repository root for full details.
*/
import { useCallback } from "react";
import { type JoinRule } from "matrix-js-sdk/src/matrix";
import type { Room } from "matrix-js-sdk/src/models/room";
import type { JoinRule, Room } from "matrix-js-sdk";
import { useRoomState } from "./useRoomState";
export function useJoinRule(room: Room): JoinRule {

18
src/room/useLoadGroupCall.ts
View File

@@ -13,18 +13,20 @@ import {
type ComponentType,
type SVGAttributes,
} from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { EventType } from "matrix-js-sdk/src/@types/event";
import {
JoinRule,
EventType,
SyncState,
MatrixError,
KnownMembership,
ClientEvent,
type MatrixClient,
type RoomSummary,
} from "matrix-js-sdk/src/client";
import { SyncState } from "matrix-js-sdk/src/sync";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { RoomEvent, type Room } from "matrix-js-sdk/src/models/room";
import { KnownMembership } from "matrix-js-sdk/src/types";
import { JoinRule, MatrixError } from "matrix-js-sdk/src/matrix";
RoomEvent,
type Room,
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { useTranslation } from "react-i18next";
import {
AdminIcon,

2
src/room/useRoomAvatar.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { useCallback } from "react";
import { type Room } from "matrix-js-sdk/src/models/room";
import { type Room } from "matrix-js-sdk";
import { useRoomState } from "./useRoomState";

2
src/room/useRoomName.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Room, RoomEvent } from "matrix-js-sdk/src/matrix";
import { type Room, RoomEvent } from "matrix-js-sdk";
import { useState } from "react";
import { useTypedEventEmitter } from "../useEvents";

6
src/room/useRoomState.ts
View File

@@ -5,13 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type RoomState,
RoomStateEvent,
} from "matrix-js-sdk/src/models/room-state";
import { useCallback, useMemo, useState } from "react";
import { type RoomState, RoomStateEvent, type Room } from "matrix-js-sdk";
import type { Room } from "matrix-js-sdk/src/models/room";
import { useTypedEventEmitter } from "../useEvents";
/**

2
src/room/useSwitchCamera.ts
View File

@@ -20,7 +20,7 @@ import {
TrackEvent,
} from "livekit-client";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { platform } from "../Platform";

5
src/rtcSessionHelpers.test.ts
View File

@@ -5,9 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { expect, onTestFinished, test, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/src/autodiscovery";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import EventEmitter from "events";
import { enterRTCSession, leaveRTCSession } from "../src/rtcSessionHelpers";
@@ -112,6 +112,7 @@ test("It joins the correct Session", async () => {
manageMediaKeys: false,
useLegacyMemberEvents: false,
useNewMembershipManager: true,
useExperimentalToDeviceTransport: false,
},
);
});

15
src/rtcSessionHelpers.ts
View File

@@ -5,15 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import {
isLivekitFocus,
isLivekitFocusConfig,
type LivekitFocus,
type LivekitFocusActive,
} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
import { AutoDiscovery } from "matrix-js-sdk/src/autodiscovery";
} from "matrix-js-sdk/lib/matrixrtc";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { PosthogAnalytics } from "./analytics/PosthogAnalytics";
import { Config } from "./config/Config";
@@ -98,6 +98,7 @@ export async function enterRTCSession(
rtcSession: MatrixRTCSession,
encryptMedia: boolean,
useNewMembershipManager = true,
useExperimentalToDeviceTransport = false,
): Promise<void> {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -120,11 +121,11 @@ export async function enterRTCSession(
...(useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents: !useDeviceSessionMemberEvents,
}),
membershipServerSideExpiryTimeout:
delayedLeaveEventDelayMs:
matrixRtcSessionConfig?.membership_server_side_expiry_timeout,
membershipKeepAlivePeriod:
matrixRtcSessionConfig?.membership_keep_alive_period,
networkErrorRetryMs: matrixRtcSessionConfig?.membership_keep_alive_period,
makeKeyDelay: matrixRtcSessionConfig?.key_rotation_on_leave_delay,
useExperimentalToDeviceTransport,
},
);
if (widget) {

66
src/settings/DeveloperSettingsTab.tsx
View File

@@ -15,11 +15,15 @@ import {
debugTileLayout as debugTileLayoutSetting,
showNonMemberTiles as showNonMemberTilesSetting,
showConnectionStats as showConnectionStatsSetting,
useNewMembershipManagerSetting,
useNewMembershipManager as useNewMembershipManagerSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
muteAllAudio as muteAllAudioSetting,
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
} from "./settings";
import type { MatrixClient } from "matrix-js-sdk/src/client";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
import styles from "./DeveloperSettingsTab.module.css";
import { useUrlParams } from "../UrlParams";
interface Props {
client: MatrixClient;
livekitRoom?: LivekitRoom;
@@ -43,6 +47,18 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
useNewMembershipManagerSetting,
);
const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
alwaysShowIphoneEarpieceSetting,
);
const [
useExperimentalToDeviceTransport,
setUseExperimentalToDeviceTransport,
] = useSetting(useExperimentalToDeviceTransportSetting);
const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting);
const urlParams = useUrlParams();
const sfuUrl = useMemo((): URL | null => {
if (livekitRoom?.engine.client.ws?.url) {
// strip the URL params
@@ -153,6 +169,48 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
)}
/>
</FieldRow>
<FieldRow>
<InputField
id="useToDeviceKeyTransport"
type="checkbox"
label={t("developer_mode.use_to_device_key_transport")}
checked={!!useExperimentalToDeviceTransport}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setUseExperimentalToDeviceTransport(event.target.checked);
},
[setUseExperimentalToDeviceTransport],
)}
/>
</FieldRow>
<FieldRow>
<InputField
id="muteAllAudio"
type="checkbox"
label={t("developer_mode.mute_all_audio")}
checked={muteAllAudio}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setMuteAllAudio(event.target.checked);
},
[setMuteAllAudio],
)}
/>
</FieldRow>{" "}
<FieldRow>
<InputField
id="alwaysShowIphoneEarpiece"
type="checkbox"
label={t("developer_mode.always_show_iphone_earpiece")}
checked={alwaysShowIphoneEarpiece}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setAlwaysShowIphoneEarpiece(event.target.checked);
},
[setAlwaysShowIphoneEarpiece],
)}
/>{" "}
</FieldRow>
{livekitRoom ? (
<>
<p>
@@ -169,6 +227,10 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
</pre>
</>
) : null}
<p>{t("developer_mode.environment_variables")}</p>
<pre>{JSON.stringify(import.meta.env, null, 2)}</pre>
<p>{t("developer_mode.url_params")}</p>
<pre>{JSON.stringify(urlParams, null, 2)}</pre>
</>
);
};

30
src/settings/DeviceSelection.tsx
View File

@@ -22,17 +22,20 @@ import {
} from "@vector-im/compound-web";
import { Trans, useTranslation } from "react-i18next";
import { type MediaDevice } from "../livekit/MediaDevicesContext";
import {
EARPIECE_CONFIG_ID,
type MediaDeviceHandle,
} from "../livekit/MediaDevicesContext";
import styles from "./DeviceSelection.module.css";
interface Props {
devices: MediaDevice;
device: MediaDeviceHandle;
title: string;
numberedLabel: (number: number) => string;
}
export const DeviceSelection: FC<Props> = ({
devices,
device,
title,
numberedLabel,
}) => {
@@ -40,12 +43,13 @@ export const DeviceSelection: FC<Props> = ({
const groupId = useId();
const onChange = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
devices.select(e.target.value);
device.select(e.target.value);
},
[devices],
[device],
);
if (devices.available.size == 0) return null;
// There is no need to show the menu if there is no choice that can be made.
if (device.available.size <= 1) return null;
return (
<div className={styles.selection}>
@@ -60,7 +64,7 @@ export const DeviceSelection: FC<Props> = ({
</Heading>
<Separator className={styles.separator} />
<div className={styles.options}>
{[...devices.available].map(([id, label]) => {
{[...device.available].map(([id, label]) => {
let labelText: ReactNode;
switch (label.type) {
case "name":
@@ -85,6 +89,16 @@ export const DeviceSelection: FC<Props> = ({
</Trans>
);
break;
case "earpiece":
labelText = t("settings.devices.earpiece");
break;
}
let isSelected = false;
if (device.useAsEarpiece) {
isSelected = id === EARPIECE_CONFIG_ID;
} else {
isSelected = id === device.selectedId;
}
return (
@@ -93,7 +107,7 @@ export const DeviceSelection: FC<Props> = ({
name={groupId}
control={
<RadioControl
checked={id === devices.selectedId}
checked={isSelected}
onChange={onChange}
value={id}
/>

4
src/settings/FeedbackSettingsTab.tsx
View File

@@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details.
*/
import { type ChangeEvent, type FC, useCallback } from "react";
import { secureRandomString } from "matrix-js-sdk/src/randomstring";
import { secureRandomString } from "matrix-js-sdk/lib/randomstring";
import { Trans, useTranslation } from "react-i18next";
import { Button, Text } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { FieldRow, InputField, ErrorMessage } from "../input/Input";
import { useSubmitRageshake, useRageshakeRequest } from "./submit-rageshake";

4
src/settings/ProfileSettingsTab.tsx
View File

@@ -6,9 +6,9 @@ Please see LICENSE in the repository root for full details.
*/
import { type FC, useCallback, useEffect, useMemo, useRef } from "react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixClient } from "matrix-js-sdk";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { useProfile } from "../profile/useProfile";
import { FieldRow, InputField, ErrorMessage } from "../input/Input";

2
src/settings/RageshakeButton.tsx
View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { useTranslation } from "react-i18next";
import { type FC, useCallback, type JSX } from "react";
import { Button } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { Config } from "../config/Config";
import styles from "./RageshakeButton.module.css";

106
src/settings/SettingsModal.tsx
View File

@@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, useState } from "react";
import { type FC, type ReactNode, useState } from "react";
import { useTranslation } from "react-i18next";
import { type MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form } from "@vector-im/compound-web";
import { type MatrixClient } from "matrix-js-sdk";
import { Button, Root as Form, Separator } from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { useObservableEagerState } from "observable-hooks";
import { Modal } from "../Modal";
import styles from "./SettingsModal.module.css";
@@ -19,18 +20,23 @@ import { FeedbackSettingsTab } from "./FeedbackSettingsTab";
import {
useMediaDevices,
useMediaDeviceNames,
iosDeviceMenu$,
} from "../livekit/MediaDevicesContext";
import { widget } from "../widget";
import {
useSetting,
soundEffectVolumeSetting,
soundEffectVolume as soundEffectVolumeSetting,
backgroundBlur as backgroundBlurSetting,
developerMode,
} from "./settings";
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
import { Slider } from "../Slider";
import { DeviceSelection } from "./DeviceSelection";
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
import { isRageshakeAvailable } from "./submit-rageshake";
import { FieldRow, InputField } from "../input/Input";
import { useSubmitRageshake } from "./submit-rageshake";
import { useUrlParams } from "../UrlParams";
type SettingsTab =
| "audio"
@@ -64,31 +70,83 @@ export const SettingsModal: FC<Props> = ({
}) => {
const { t } = useTranslation();
// Generate a `Checkbox` input to turn blur on or off.
const BlurCheckbox: React.FC = (): ReactNode => {
const { supported } = useTrackProcessor();
const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
return (
<>
<h4>{t("settings.background_blur_header")}</h4>
<FieldRow>
<InputField
id="activateBackgroundBlur"
label={t("settings.background_blur_label")}
description={
supported ? "" : t("settings.blur_not_supported_by_browser")
}
type="checkbox"
checked={!!blurActive}
onChange={(b): void => setBlurActive(b.target.checked)}
disabled={!supported}
/>
</FieldRow>
</>
);
};
const devices = useMediaDevices();
useMediaDeviceNames(devices, open);
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
const [showDeveloperSettingsTab] = useSetting(developerMode);
const { available: isRageshakeAvailable } = useSubmitRageshake();
// For controlled devices, we will not show the input section:
// Controlled media devices are used on mobile platforms, where input and output are grouped into
// a single device. These are called "headset" or "speaker" (or similar) but contain both input and output.
// On EC, we decided that it is less confusing for the user if they see those options in the output section
// rather than the input section.
const { controlledAudioDevices } = useUrlParams();
// If we are on iOS we will show a button to open the native audio device picker.
const iosDeviceMenu = useObservableEagerState(iosDeviceMenu$);
const audioTab: Tab<SettingsTab> = {
key: "audio",
name: t("common.audio"),
content: (
<>
<Form>
{!controlledAudioDevices && (
<DeviceSelection
device={devices.audioInput}
title={t("settings.devices.microphone")}
numberedLabel={(n) =>
t("settings.devices.microphone_numbered", { n })
}
/>
)}
{iosDeviceMenu && (
<Button
onClick={(e): void => {
e.preventDefault();
window.controls.showNativeAudioDevicePicker?.();
// call deprecated method for backwards compatibility.
window.controls.showNativeOutputDevicePicker?.();
}}
>
{t("settings.devices.change_device_button")}
</Button>
)}
<DeviceSelection
devices={devices.audioInput}
title={t("settings.devices.microphone")}
numberedLabel={(n) =>
t("settings.devices.microphone_numbered", { n })
}
/>
<DeviceSelection
devices={devices.audioOutput}
device={devices.audioOutput}
title={t("settings.devices.speaker")}
numberedLabel={(n) => t("settings.devices.speaker_numbered", { n })}
/>
<div className={styles.volumeSlider}>
<label>{t("settings.audio_tab.effect_volume_label")}</label>
<p>{t("settings.audio_tab.effect_volume_description")}</p>
@@ -111,13 +169,17 @@ export const SettingsModal: FC<Props> = ({
key: "video",
name: t("common.video"),
content: (
<Form>
<DeviceSelection
devices={devices.videoInput}
title={t("settings.devices.camera")}
numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
/>
</Form>
<>
<Form>
<DeviceSelection
device={devices.videoInput}
title={t("settings.devices.camera")}
numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
/>
</Form>
<Separator />
<BlurCheckbox />
</>
),
};
@@ -148,7 +210,7 @@ export const SettingsModal: FC<Props> = ({
const tabs = [audioTab, videoTab];
if (widget === null) tabs.push(profileTab);
tabs.push(preferencesTab);
if (isRageshakeAvailable() || import.meta.env.VITE_PACKAGE === "full") {
if (isRageshakeAvailable || import.meta.env.VITE_PACKAGE === "full") {
// for full package we want to show the analytics consent checkbox
// even if rageshake is not available
tabs.push(feedbackTab);

9
src/settings/rageshake.ts
View File

@@ -29,8 +29,8 @@ Please see LICENSE in the repository root for full details.
import EventEmitter from "events";
import { throttle } from "lodash-es";
import { type Logger, logger } from "matrix-js-sdk/src/logger";
import { secureRandomString } from "matrix-js-sdk/src/randomstring";
import { type Logger, logger } from "matrix-js-sdk/lib/logger";
import { secureRandomString } from "matrix-js-sdk/lib/randomstring";
import { type LoggingMethod } from "loglevel";
import type loglevel from "loglevel";
@@ -473,11 +473,6 @@ export async function init(): Promise<void> {
// configure loglevel based loggers:
setLogExtension(logger, global.mx_rage_logger.log);
// these are the child/prefixed loggers we want to capture from js-sdk
// there doesn't seem to be an easy way to capture all children
["MatrixRTCSession", "MatrixRTCSessionManager"].forEach((loggerName) => {
setLogExtension(logger.getChild(loggerName), global.mx_rage_logger.log);
});
// intercept console logging so that we can get matrix_sdk logs:
// this is nasty, but no logging hooks are provided

24
src/settings/settings.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import { BehaviorSubject, type Observable } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
@@ -44,6 +44,9 @@ export class Setting<T> {
this._value$.next(value);
localStorage.setItem(this.key, JSON.stringify(value));
};
public readonly getValue = (): T => {
return this._value$.getValue();
};
}
/**
@@ -96,6 +99,8 @@ export const videoInput = new Setting<string | undefined>(
undefined,
);
export const backgroundBlur = new Setting<boolean>("background-blur", false);
export const showHandRaisedTimer = new Setting<boolean>(
"hand-raised-show-timer",
false,
@@ -108,13 +113,26 @@ export const playReactionsSound = new Setting<boolean>(
true,
);
export const soundEffectVolumeSetting = new Setting<number>(
export const soundEffectVolume = new Setting<number>(
"sound-effect-volume",
0.5,
);
export const useNewMembershipManagerSetting = new Setting<boolean>(
export const useNewMembershipManager = new Setting<boolean>(
"new-membership-manager",
true,
);
export const useExperimentalToDeviceTransport = new Setting<boolean>(
"experimental-to-device-transport",
true,
);
export const muteAllAudio = new Setting<boolean>("mute-all-audio", false);
export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);
export const alwaysShowIphoneEarpiece = new Setting<boolean>(
"always-show-iphone-earpiece",
false,
);

68
src/settings/submit-rageshake.test.ts
View File

@@ -15,78 +15,12 @@ import {
beforeEach,
} from "vitest";
import {
getRageshakeSubmitUrl,
isRageshakeAvailable,
} from "./submit-rageshake";
import { getRageshakeSubmitUrl } from "./submit-rageshake";
import { getUrlParams } from "../UrlParams";
import { mockConfig } from "../utils/test";
vi.mock("../UrlParams", () => ({ getUrlParams: vi.fn() }));
describe("isRageshakeAvailable", () => {
beforeEach(() => {
(getUrlParams as Mock).mockReturnValue({});
mockConfig({});
});
afterEach(() => {
vi.unstubAllEnvs();
vi.clearAllMocks();
});
describe("embedded package", () => {
beforeEach(() => {
vi.stubEnv("VITE_PACKAGE", "embedded");
});
it("returns false with no rageshakeSubmitUrl URL param", () => {
expect(isRageshakeAvailable()).toBe(false);
});
it("ignores config value and returns false with no rageshakeSubmitUrl URL param", () => {
mockConfig({
rageshake: {
submit_url: "https://config.example.com.localhost",
},
});
expect(isRageshakeAvailable()).toBe(false);
});
it("returns true with rageshakeSubmitUrl URL param", () => {
(getUrlParams as Mock).mockReturnValue({
rageshakeSubmitUrl: "https://url.example.com.localhost",
});
expect(isRageshakeAvailable()).toBe(true);
});
});
describe("full package", () => {
beforeEach(() => {
vi.stubEnv("VITE_PACKAGE", "full");
});
it("returns false with no config value", () => {
expect(isRageshakeAvailable()).toBe(false);
});
it("ignores rageshakeSubmitUrl URL param and returns false with no config value", () => {
(getUrlParams as Mock).mockReturnValue({
rageshakeSubmitUrl: "https://url.example.com.localhost",
});
expect(isRageshakeAvailable()).toBe(false);
});
it("returns true with config value", () => {
mockConfig({
rageshake: {
submit_url: "https://config.example.com.localhost",
},
});
expect(isRageshakeAvailable()).toBe(true);
});
});
});
describe("getRageshakeSubmitUrl", () => {
beforeEach(() => {
(getUrlParams as Mock).mockReturnValue({});

23
src/settings/submit-rageshake.ts
View File

@@ -6,13 +6,13 @@ Please see LICENSE in the repository root for full details.
*/
import { type ComponentProps, useCallback, useEffect, useState } from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ClientEvent,
type MatrixClient,
type MatrixEvent,
} from "matrix-js-sdk/src/matrix";
import { type CryptoApi } from "matrix-js-sdk/src/crypto-api";
} from "matrix-js-sdk";
import { type CryptoApi } from "matrix-js-sdk/lib/crypto-api";
import { getLogsForReport } from "./rageshake";
import { useClient } from "../ClientContext";
@@ -131,11 +131,9 @@ export function getRageshakeSubmitUrl(): string | undefined {
return undefined;
}
export function isRageshakeAvailable(): boolean {
return !!getRageshakeSubmitUrl();
}
export function useSubmitRageshake(): {
export function useSubmitRageshake(
injectedGetRageshakeSubmitUrl = getRageshakeSubmitUrl,
): {
submitRageshake: (opts: RageShakeSubmitOptions) => Promise<void>;
sending: boolean;
sent: boolean;
@@ -158,7 +156,8 @@ export function useSubmitRageshake(): {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
async (opts) => {
if (!getRageshakeSubmitUrl()) {
const submitUrl = injectedGetRageshakeSubmitUrl();
if (!submitUrl) {
throw new Error("No rageshake URL is configured");
}
@@ -292,7 +291,7 @@ export function useSubmitRageshake(): {
);
}
const res = await fetch(Config.get().rageshake!.submit_url, {
const res = await fetch(submitUrl, {
method: "POST",
body,
});
@@ -309,7 +308,7 @@ export function useSubmitRageshake(): {
logger.error(error);
}
},
[client, sending],
[client, sending, injectedGetRageshakeSubmitUrl],
);
return {
@@ -317,7 +316,7 @@ export function useSubmitRageshake(): {
sending,
sent,
error,
available: isRageshakeAvailable(),
available: !!injectedGetRageshakeSubmitUrl(),
};
}

Some files were not shown because too many files have changed in this diff Show More