studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep tiles in a stable order (#2670)

Browse Source 
* Keep tiles in a stable order

This introduces a new layer of abstraction on top of MediaViewModel: TileViewModel, which gives us a place to store data relating to tiles rather than their media, and also generally makes it easier to reason about tiles as they move about the call layout. I have created a class called TileStore to keep track of these tiles.

This allows us to swap out the media shown on a tile as the spotlight speaker changes, and avoid moving tiles around unless they really need to jump between the visible/invisible regions of the layout.

* Don't throttle spotlight updates

Since we now assume that the spotlight and grid will be in sync (i.e. an active speaker in one will behave as an active speaker in the other), we don't want the spotlight to ever lag behind due to throttling. If this causes usability issues we should maybe look into making LiveKit's 'speaking' indicators less erratic first.

* Make layout shifts due to a change in speaker less surprising

Although we try now to avoid layout shifts due to the spotlight speaker changing wherever possible, a spotlight speaker coming from off screen can still trigger one. Let's shift the layout a bit more gracefully in this case.

* Improve the tile ordering tests

* Maximize the spotlight tile in portrait layout

* Tell tiles whether they're actually visible in a more timely manner

* Fix test

* Fix speaking indicators logic

* Improve readability of marbles

* Fix test case

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
Robin
2024-11-06 04:36:48 -05:00
committed by GitHub
parent 22cca2874a
commit d3f069e763
22 changed files with 1178 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/grid/CallLayout.ts
View File

@@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject, Observable } from "rxjs";
import { ComponentType } from "react";
import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel";
import { LayoutProps } from "./Grid";
import { TileViewModel } from "../state/TileViewModel";
export interface Bounds {
width: number;
@@ -42,19 +42,6 @@ export interface CallLayoutInputs {
pipAlignment: BehaviorSubject<Alignment>;
}
export interface GridTileModel {
type: "grid";
vm: UserMediaViewModel;
}
export interface SpotlightTileModel {
type: "spotlight";
vms: MediaViewModel[];
maximised: boolean;
}
export type TileModel = GridTileModel | SpotlightTileModel;
export interface CallLayoutOutputs<Model> {
/**
* Whether the scrolling layer of the layout should appear on top.
@@ -63,11 +50,11 @@ export interface CallLayoutOutputs<Model> {
/**
* The visually fixed (non-scrolling) layer of the layout.
*/
fixed: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
fixed: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
/**
* The layer of the layout that can overflow and be scrolled.
*/
scrolling: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
scrolling: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
}
/**

50
src/grid/Grid.tsx
View File

@@ -24,6 +24,7 @@ import {
createContext,
forwardRef,
memo,
useCallback,
useContext,
useEffect,
useMemo,
@@ -33,6 +34,8 @@ import {
import useMeasure from "react-use-measure";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { useObservableEagerState } from "observable-hooks";
import { fromEvent, map, startWith } from "rxjs";
import styles from "./Grid.module.css";
import { useMergedRefs } from "../useMergedRefs";
@@ -51,6 +54,7 @@ interface Tile<Model> {
id: string;
model: Model;
onDrag: DragCallback | undefined;
setVisible: (visible: boolean) => void;
}
type PlacedTile<Model> = Tile<Model> & Rect;
@@ -84,6 +88,7 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
onVisibilityChange?: (visible: boolean) => void;
style?: CSSProperties;
className?: string;
}
@@ -131,6 +136,11 @@ export function useUpdateLayout(): void {
);
}
const windowHeightObservable = fromEvent(window, "resize").pipe(
startWith(null),
map(() => window.innerHeight),
);
export interface LayoutProps<LayoutModel, TileModel, R extends HTMLElement> {
ref: LegacyRef<R>;
model: LayoutModel;
@@ -232,6 +242,7 @@ export function Grid<
const [gridRoot, gridRef2] = useState<HTMLElement | null>(null);
const gridRef = useMergedRefs<HTMLElement>(gridRef1, gridRef2);
const windowHeight = useObservableEagerState(windowHeightObservable);
const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
const [generation, setGeneration] = useState<number | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
@@ -239,12 +250,34 @@ export function Grid<
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
function Slot({ id, model, onDrag, style, className, ...props }) {
function Slot({
id,
model,
onDrag,
onVisibilityChange,
style,
className,
...props
}) {
const ref = useRef<HTMLDivElement | null>(null);
const prevVisible = useRef<boolean | null>(null);
const setVisible = useCallback(
(visible: boolean) => {
if (
onVisibilityChange !== undefined &&
visible !== prevVisible.current
) {
onVisibilityChange(visible);
prevVisible.current = visible;
}
},
[onVisibilityChange],
);
useEffect(() => {
tiles.set(id, { id, model, onDrag });
tiles.set(id, { id, model, onDrag, setVisible });
return (): void => void tiles.delete(id);
}, [id, model, onDrag]);
}, [id, model, onDrag, setVisible]);
return (
<div
@@ -302,6 +335,17 @@ export function Grid<
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [gridRoot, layoutRoot, tiles, gridBounds, generation]);
// The height of the portion of the grid visible at any given time
const visibleHeight = useMemo(
() => Math.min(gridBounds.bottom, windowHeight) - gridBounds.top,
[gridBounds, windowHeight],
);
useEffect(() => {
for (const tile of placedTiles)
tile.setVisible(tile.y + tile.height <= visibleHeight);
}, [placedTiles, visibleHeight]);
// Drag state is stored in a ref rather than component state, because we use
// react-spring's imperative API during gestures to improve responsiveness
const dragState = useRef<DragState | null>(null);

35
src/grid/GridLayout.tsx
View File

@@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks";
import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
interface GridCSSProperties extends CSSProperties {
@@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
),
),
);
const tileModel: TileModel | undefined = useMemo(
() =>
model.spotlight && {
type: "spotlight",
vms: model.spotlight,
maximised: false,
},
[model.spotlight],
);
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) =>
@@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
return (
<div ref={ref} className={styles.fixed}>
{tileModel && (
{model.spotlight && (
<Slot
className={styles.slot}
id="spotlight"
model={tileModel}
model={model.spotlight}
onDrag={onDragSpotlight}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
@@ -93,11 +79,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
[width, minHeight, model.grid.length],
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
return (
<div
ref={ref}
@@ -111,8 +92,14 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
} as GridCSSProperties
}
>
{tileModels.map((m) => (
<Slot key={m.vm.id} className={styles.slot} id={m.vm.id} model={m} />
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>
);

21
src/grid/OneOnOneLayout.tsx
View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneLayout.module.css";
import { DragCallback, useUpdateLayout } from "./Grid";
@@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
[width, height],
);
const remoteTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.remote }),
[model.remote],
);
const localTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.local }),
[model.local],
);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
return (
<div ref={ref} className={styles.layer}>
<Slot
id={remoteTileModel.vm.id}
model={remoteTileModel}
id={model.remote.id}
model={model.remote}
onVisibilityChange={model.remote.setVisible}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
<Slot
className={classNames(styles.slot, styles.local)}
id={localTileModel.vm.id}
model={localTileModel}
id={model.local.id}
model={model.local}
onDrag={onDragLocalTile}
onVisibilityChange={model.local.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

22
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useCallback, useMemo } from "react";
import { forwardRef, useCallback } from "react";
import { useObservableEagerState } from "observable-hooks";
import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";
@@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
ref,
) {
useUpdateLayout();
const spotlightTileModel: SpotlightTileModel = useMemo(
() => ({ type: "spotlight", vms: model.spotlight, maximised: true }),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<Slot
className={styles.spotlight}
id="spotlight"
model={spotlightTileModel}
model={model.spotlight}
/>
</div>
);
@@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
useUpdateLayout();
const pipAlignmentValue = useObservableEagerState(pipAlignment);
const pipTileModel: GridTileModel | undefined = useMemo(
() => model.pip && { type: "grid", vm: model.pip },
[model.pip],
);
const onDragPip: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
return (
<div ref={ref} className={styles.layer}>
{pipTileModel && (
{model.pip && (
<Slot
className={styles.pip}
id="pip"
model={pipTileModel}
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
onVisibilityChange={model.pip.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

33
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useMemo } from "react";
import { forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { CallLayout, GridTileModel, TileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -30,19 +30,15 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: false,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
<div className={styles.grid} />
</div>
@@ -55,25 +51,24 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div ref={ref} className={styles.layer}>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>

38
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -5,16 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { CSSProperties, forwardRef, useMemo } from "react";
import { CSSProperties, forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -40,19 +35,15 @@ export const makeSpotlightPortraitLayout: CallLayout<
ref,
) {
useUpdateLayout();
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: true,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
</div>
);
@@ -71,10 +62,8 @@ export const makeSpotlightPortraitLayout: CallLayout<
width,
model.grid.length,
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div
@@ -90,16 +79,17 @@ export const makeSpotlightPortraitLayout: CallLayout<
>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>