studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

almost mvp

Browse Source
This commit is contained in:
Timo K
2025-11-30 20:31:21 +01:00
parent 28158bfc23
commit 2d8ffc0ccd
11 changed files with 1144 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
godot/README.md
View File

@@ -0,0 +1,14 @@
## url parameters
widgetId = $matrix_widget_id
perParticipantE2EE = true
userId = $matrix_user_id
deviceId = $org.matrix.msc3819.matrix_device_id
baseUrl = $org.matrix.msc4039.matrix_base_url
parentUrl = // will be inserted automatically
http://localhost?widgetId=&perParticipantE2EE=true&userId=&deviceId=&baseUrl=&roomId=
->
http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id

BIN
godot/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.4 KiB

51
godot/helper.ts Normal file
View File

@@ -0,0 +1,51 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { scan } from "rxjs";
import { widget as _widget } from "../src/widget";
import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
export const logger = rootLogger.getChild("[MatrixRTCSdk]");
if (!_widget) throw Error("No widget. This webapp can only start as a widget");
export const widget = _widget;
export const tryMakeSticky = (): void => {
logger.info("try making sticky MatrixRTCSdk");
void widget.api
.setAlwaysOnScreen(true)
.then(() => {
logger.info("sticky MatrixRTCSdk");
})
.catch((error) => {
logger.error("failed to make sticky MatrixRTCSdk", error);
});
};
export const TEXT_LK_TOPIC = "matrixRTC";
/**
* simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
*
* I think there should be a builtin for this but i did not find it...
*/
export const currentAndPrev = scan<
LivekitRoomItem[],
{
prev: LivekitRoomItem[];
current: LivekitRoomItem[];
}
>(
({ current: lastCurrentVal }, items) => ({
prev: lastCurrentVal,
current: items,
}),
{
prev: [],
current: [],
},
);

45
godot/index.html Normal file
View File

@@ -0,0 +1,45 @@
<!doctype html>
<html>
<head>
<title>Godot MatrixRTC Widget</title>
<meta charset="utf-8" />
<script type="module">
import { createMatrixRTCSdk } from "http://localhost:8123/matrixrtc-ec-godot.js";
try {
console.log("Hello from index.html");
try {
window.matrixRTCSdk = await createMatrixRTCSdk();
console.info("createMatrixRTCSdk was created!");
} catch (e) {
console.error("createMatrixRTCSdk", e);
}
// const sdk = window.matrixRTCSdk;
console.info("matrixRTCSdk join ", window.matrixRTCSdk);
await window.matrixRTCSdk.join();
console.info("matrixRTCSdk joined ");
// sdk.data$.subscribe((data) => {
// console.log(data);
// const div = document.getElementById("data");
// div.appendChild(document.createTextNode(data));
// // TODO forward to godot
// });
// var engine = new Engine($GODOT_CONFIG);
// engine.startGame();
} catch (e) {
console.error("catchALL,", e);
}
</script>
<!--<script src="$GODOT_URL"></script>-->
</head>
<body>
<button onclick="window.matrixRTCSdk.leave();">Leave</button>
<!--<button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});">
Send Text
</button>-->
<div id="data"></div>
<canvas id="canvas"></canvas>
</body>
</html>

236
godot/main.ts
View File

@@ -4,42 +4,57 @@ Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { of } from "rxjs";
import { loadClient } from "../src/ClientContext.tsx"; // import { type InitResult } from "../src/ClientContext";
import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel.ts"; import { map, type Observable, of, Subject, switchMap } from "rxjs";
import { MuteStates } from "../src/state/MuteStates.ts"; import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc";
import { ObservableScope } from "../src/state/ObservableScope.ts"; import { type TextStreamInfo } from "livekit-client/dist/src/room/types";
import { getUrlParams } from "../src/UrlParams.ts"; import {
type Room as LivekitRoom,
type TextStreamReader,
} from "livekit-client";
import { type Behavior, constant } from "../src/state/Behavior";
import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel";
import { ObservableScope } from "../src/state/ObservableScope";
import { getUrlParams } from "../src/UrlParams";
import { MuteStates } from "../src/state/MuteStates";
import { MediaDevices } from "../src/state/MediaDevices"; import { MediaDevices } from "../src/state/MediaDevices";
import { constant } from "../src/state/Behavior.ts"; import { E2eeType } from "../src/e2ee/e2eeType";
import { E2eeType } from "../src/e2ee/e2eeType.ts"; import { type LocalMemberConnectionState } from "../src/state/CallViewModel/localMember/LocalMembership";
import {
currentAndPrev,
logger,
TEXT_LK_TOPIC,
tryMakeSticky,
widget,
} from "./helper";
import { ElementWidgetActions } from "../src/widget";
console.log("test Godot EC export"); interface MatrixRTCSdk {
join: () => LocalMemberConnectionState;
export async function start(): Promise<void> { /** @throws on leave errors */
const initResults = await loadClient(); leave: () => void;
if (initResults === null) { data$: Observable<{ sender: string; data: string }>;
console.error("could not init client"); sendData?: (data: Record<string, unknown>) => Promise<void>;
return; }
} export async function createMatrixRTCSdk(): Promise<MatrixRTCSdk> {
const { client } = initResults; logger.info("Hello");
const client = await widget.client;
logger.info("client created");
const scope = new ObservableScope(); const scope = new ObservableScope();
const { roomId } = getUrlParams(); const { roomId } = getUrlParams();
if (roomId === null) { if (roomId === null) throw Error("could not get roomId from url params");
console.error("could not get roomId from url params");
return;
}
const room = client.getRoom(roomId); const room = client.getRoom(roomId);
if (room === null) { if (room === null) throw Error("could not get room from client");
console.error("could not get room from client");
return;
}
const mediaDevices = new MediaDevices(scope); const mediaDevices = new MediaDevices(scope);
const muteStates = new MuteStates(scope, mediaDevices, constant(true)); const muteStates = new MuteStates(scope, mediaDevices, constant(true));
const rtcSession = client.matrixRTC.getRoomSession(room);
const callViewModel = createCallViewModel$( const callViewModel = createCallViewModel$(
scope, scope,
client.matrixRTC.getRoomSession(room), rtcSession,
room, room,
mediaDevices, mediaDevices,
muteStates, muteStates,
@@ -48,29 +63,148 @@ export async function start(): Promise<void> {
of({}), of({}),
constant({ supported: false, processor: undefined }), constant({ supported: false, processor: undefined }),
); );
callViewModel.join(); logger.info("CallViewModelCreated");
// callViewModel.audioParticipants$.pipe( // create data listener
// switchMap((lkRooms) => { const data$ = new Subject<{ sender: string; data: string }>();
// for (const item of lkRooms) {
// item.livekitRoom.registerTextStreamHandler;
// }
// }),
// );
}
// Example default godot export
// <!DOCTYPE html> // const lkTextStreamHandlerFunction = async (
// <html> // reader: TextStreamReader,
// <head> // participantInfo: { identity: string },
// <title>My Template</title> // livekitRoom: LivekitRoom,
// <meta charset="UTF-8"> // ): Promise<void> => {
// </head> // const info = reader.info;
// <body> // console.log(
// <canvas id="canvas"></canvas> // `Received text stream from ${participantInfo.identity}\n` +
// <script src="$GODOT_URL"></script> // ` Topic: ${info.topic}\n` +
// <script> // ` Timestamp: ${info.timestamp}\n` +
// var engine = new Engine($GODOT_CONFIG); // ` ID: ${info.id}\n` +
// engine.startGame(); // ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
// </script> // );
// </body>
// </html> // const participants = callViewModel.livekitRoomItems$.value.find(
// (i) => i.livekitRoom === livekitRoom,
// )?.participants;
// if (participants && participants.includes(participantInfo.identity)) {
// const text = await reader.readAll();
// console.log(`Received text: ${text}`);
// data$.next({ sender: participantInfo.identity, data: text });
// } else {
// logger.warn(
// "Received text from unknown participant",
// participantInfo.identity,
// );
// }
// };
// const livekitRoomItemsSub = callViewModel.livekitRoomItems$
// .pipe(currentAndPrev)
// .subscribe({
// next: ({ prev, current }) => {
// const prevRooms = prev.map((i) => i.livekitRoom);
// const currentRooms = current.map((i) => i.livekitRoom);
// const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
// const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
// addedRooms.forEach((r) =>
// r.registerTextStreamHandler(
// TEXT_LK_TOPIC,
// (reader, participantInfo) =>
// void lkTextStreamHandlerFunction(reader, participantInfo, r),
// ),
// );
// removedRooms.forEach((r) =>
// r.unregisterTextStreamHandler(TEXT_LK_TOPIC),
// );
// },
// complete: () => {
// logger.info("Livekit room items subscription completed");
// for (const item of callViewModel.livekitRoomItems$.value) {
// logger.info("unregistering room item from room", item.url);
// item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
// }
// },
// });
// create sendData function
// const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
// scope.behavior(
// callViewModel.localMatrixLivekitMember$.pipe(
// switchMap((m) => {
// if (!m)
// return of((data: string): never => {
// throw Error("local membership not yet ready.");
// });
// return m.participant$.pipe(
// map((p) => {
// if (p === null) {
// return (data: string): never => {
// throw Error("local participant not yet ready to send data.");
// };
// } else {
// return async (data: string): Promise<TextStreamInfo> =>
// p.sendText(data, { topic: TEXT_LK_TOPIC });
// }
// }),
// );
// }),
// ),
// );
// const sendData = async (data: Record<string, unknown>): Promise<void> => {
// const dataString = JSON.stringify(data);
// try {
// const info = await sendFn.value(dataString);
// logger.info(`Sent text with stream ID: ${info.id}`);
// } catch (e) {
// console.error("failed sending: ", dataString, e);
// }
// };
// after hangup gets called
const leaveSubs = callViewModel.leave$.subscribe(() => {
const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
const leaveResolver = Promise.withResolvers<void>();
logger.info("waiting for RTC leave");
rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
logger.info("received RTC join update: ", isJoined);
if (!isJoined) leaveResolver.resolve();
});
await leaveResolver.promise;
logger.info("send Unstick");
await widget.api
.setAlwaysOnScreen(false)
.catch((e) =>
logger.error(
"Failed to set call widget `alwaysOnScreen` to false",
e,
),
);
logger.info("send Close");
await widget.api.transport
.send(ElementWidgetActions.Close, {})
.catch((e) => logger.error("Failed to send close action", e));
};
// schedule close first and then leave (scope.end)
void scheduleWidgetCloseOnLeave();
// actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
scope.end();
});
logger.info("createMatrixRTCSdk done");
return {
join: (): LocalMemberConnectionState => {
// first lets try making the widget sticky
tryMakeSticky();
return callViewModel.join();
},
leave: (): void => {
callViewModel.hangup();
leaveSubs.unsubscribe();
// livekitRoomItemsSub.unsubscribe();
},
data$,
// sendData,
};
}

2
package.json
View File

@@ -113,6 +113,7 @@
"loglevel": "^1.9.1", "loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21",
"matrix-widget-api": "^1.13.0", "matrix-widget-api": "^1.13.0",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1", "normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3", "observable-hooks": "^4.2.3",
"pako": "^2.0.4", "pako": "^2.0.4",
@@ -135,6 +136,7 @@
"vite": "^7.0.0", "vite": "^7.0.0",
"vite-plugin-generate-file": "^0.3.0", "vite-plugin-generate-file": "^0.3.0",
"vite-plugin-html": "^3.2.2", "vite-plugin-html": "^3.2.2",
"vite-plugin-node-stdlib-browser": "^0.2.1",
"vite-plugin-singlefile": "^2.3.0", "vite-plugin-singlefile": "^2.3.0",
"vite-plugin-svgr": "^4.0.0", "vite-plugin-svgr": "^4.0.0",
"vitest": "^3.0.0", "vitest": "^3.0.0",

2
src/room/InCallView.tsx
View File

@@ -251,7 +251,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(), () => void toggleRaisedHand(),
); );
const audioParticipants = useBehavior(vm.audioParticipants$); const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$); const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$); const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$); const windowMode = useBehavior(vm.windowMode$);

27
src/state/CallViewModel/CallViewModel.ts
View File

@@ -12,6 +12,7 @@ import {
ExternalE2EEKeyProvider, ExternalE2EEKeyProvider,
type Room as LivekitRoom, type Room as LivekitRoom,
type RoomOptions, type RoomOptions,
type LocalParticipant as LocalLivekitParticipant,
} from "livekit-client"; } from "livekit-client";
import { type Room as MatrixRoom } from "matrix-js-sdk"; import { type Room as MatrixRoom } from "matrix-js-sdk";
import { import {
@@ -174,12 +175,19 @@ interface LayoutScanState {
} }
type MediaItem = UserMedia | ScreenShare; type MediaItem = UserMedia | ScreenShare;
type AudioLivekitItem = { export type LivekitRoomItem = {
livekitRoom: LivekitRoom; livekitRoom: LivekitRoom;
participants: string[]; participants: string[];
url: string; url: string;
}; };
export type LocalMatrixLivekitMember = Pick<
MatrixLivekitMember,
"userId" | "membership$" | "connection$"
> & {
participant$: Behavior<LocalLivekitParticipant | null>;
};
/** /**
* The return of createCallViewModel$ * The return of createCallViewModel$
* this interface represents the root source of data for the call view. * this interface represents the root source of data for the call view.
@@ -197,8 +205,11 @@ export interface CallViewModel {
callPickupState$: Behavior< callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null "unknown" | "ringing" | "timeout" | "decline" | "success" | null
>; >;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
*/
leave$: Observable<"user" | AutoLeaveReason>; leave$: Observable<"user" | AutoLeaveReason>;
/** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
hangup: () => void; hangup: () => void;
// joining // joining
@@ -250,9 +261,10 @@ export interface CallViewModel {
*/ */
participantCount$: Behavior<number>; participantCount$: Behavior<number>;
/** Participants sorted by livekit room so they can be used in the audio rendering */ /** Participants sorted by livekit room so they can be used in the audio rendering */
audioParticipants$: Behavior<AudioLivekitItem[]>; livekitRoomItems$: Behavior<LivekitRoomItem[]>;
/** use the layout instead, this is just for the godot export. */ /** use the layout instead, this is just for the godot export. */
userMedia$: Behavior<UserMedia[]>; userMedia$: Behavior<UserMedia[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */ /** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>; handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -503,14 +515,14 @@ export function createCallViewModel$(
userId: userId, userId: userId,
}; };
const localMatrixLivekitMember$: Behavior<MatrixLivekitMember | null> = const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
scope.behavior( scope.behavior(
localRtcMembership$.pipe( localRtcMembership$.pipe(
switchMap((membership) => { switchMap((membership) => {
if (!membership) return of(null); if (!membership) return of(null);
return of( return of(
// casting is save here since we know that localRtcMembership$ is !== null since we reached this case. // casting is save here since we know that localRtcMembership$ is !== null since we reached this case.
localMatrixLivekitMemberUninitialized as MatrixLivekitMember, localMatrixLivekitMemberUninitialized as LocalMatrixLivekitMember,
); );
}), }),
), ),
@@ -621,7 +633,7 @@ export function createCallViewModel$(
return a$; return a$;
}), }),
map((members) => map((members) =>
members.reduce<AudioLivekitItem[]>((acc, curr) => { members.reduce<LivekitRoomItem[]>((acc, curr) => {
if (!curr) return acc; if (!curr) return acc;
const existing = acc.find((item) => item.url === curr.url); const existing = acc.find((item) => item.url === curr.url);
@@ -1477,7 +1489,7 @@ export function createCallViewModel$(
), ),
participantCount$: participantCount$, participantCount$: participantCount$,
audioParticipants$: audioParticipants$, livekitRoomItems$: audioParticipants$,
handsRaised$: handsRaised$, handsRaised$: handsRaised$,
reactions$: reactions$, reactions$: reactions$,
@@ -1498,6 +1510,7 @@ export function createCallViewModel$(
pip$: pip$, pip$: pip$,
layout$: layout$, layout$: layout$,
userMedia$, userMedia$,
localMatrixLivekitMember$,
tileStoreGeneration$: tileStoreGeneration$, tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$, showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$, showSpeakingIndicators$: showSpeakingIndicators$,

19
src/widget.ts
View File

@@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => {
try { try {
const { widgetId, parentUrl } = getUrlParams(); const { widgetId, parentUrl } = getUrlParams();
const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } =
getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
if (widgetId && parentUrl) { if (widgetId && parentUrl) {
const parentOrigin = new URL(parentUrl).origin; const parentOrigin = new URL(parentUrl).origin;
logger.info("Widget API is available"); logger.info("Widget API is available");
@@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => {
// We need to do this now rather than later because it has capabilities to // We need to do this now rather than later because it has capabilities to
// request, and is responsible for starting the transport (should it be?) // request, and is responsible for starting the transport (should it be?)
const {
roomId,
userId,
deviceId,
baseUrl,
e2eEnabled,
allowIceFallback,
} = getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
// These are all the event types the app uses // These are all the event types the app uses
const sendEvent = [ const sendEvent = [
EventType.CallNotify, // Sent as a deprecated fallback EventType.CallNotify, // Sent as a deprecated fallback

3
vite-godot.config.js
View File

@@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details.
*/ */
import { defineConfig, mergeConfig } from "vite"; import { defineConfig, mergeConfig } from "vite";
import { viteSingleFile } from "vite-plugin-singlefile";
import fullConfig from "./vite.config"; import fullConfig from "./vite.config";
import nodePolyfills from "vite-plugin-node-stdlib-browser";
const base = "./"; const base = "./";
@@ -27,6 +27,7 @@ export default defineConfig((env) =>
fileName: "matrixrtc-ec-godot", fileName: "matrixrtc-ec-godot",
}, },
}, },
plugins: [nodePolyfills()],
}), }),
), ),
); );

845
yarn.lock
View File

File diff suppressed because it is too large Load Diff