studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep tiles in a stable order (#2670)

Browse Source 
* Keep tiles in a stable order

This introduces a new layer of abstraction on top of MediaViewModel: TileViewModel, which gives us a place to store data relating to tiles rather than their media, and also generally makes it easier to reason about tiles as they move about the call layout. I have created a class called TileStore to keep track of these tiles.

This allows us to swap out the media shown on a tile as the spotlight speaker changes, and avoid moving tiles around unless they really need to jump between the visible/invisible regions of the layout.

* Don't throttle spotlight updates

Since we now assume that the spotlight and grid will be in sync (i.e. an active speaker in one will behave as an active speaker in the other), we don't want the spotlight to ever lag behind due to throttling. If this causes usability issues we should maybe look into making LiveKit's 'speaking' indicators less erratic first.

* Make layout shifts due to a change in speaker less surprising

Although we try now to avoid layout shifts due to the spotlight speaker changing wherever possible, a spotlight speaker coming from off screen can still trigger one. Let's shift the layout a bit more gracefully in this case.

* Improve the tile ordering tests

* Maximize the spotlight tile in portrait layout

* Tell tiles whether they're actually visible in a more timely manner

* Fix test

* Fix speaking indicators logic

* Improve readability of marbles

* Fix test case

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
Robin
2024-11-06 04:36:48 -05:00
committed by GitHub
parent 22cca2874a
commit d3f069e763
22 changed files with 1178 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
src/state/CallViewModel.ts
View File

@@ -46,7 +46,6 @@ import {
switchMap,
switchScan,
take,
throttleTime,
timer,
withLatestFrom,
} from "rxjs";
@@ -70,6 +69,12 @@ import { ObservableScope } from "./ObservableScope";
import { duplicateTiles } from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled } from "../controls";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
// How long we wait after a focus switch before showing the real participant
@@ -80,39 +85,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
export interface GridLayout {
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayout {
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayout {
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayout {
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: MediaViewModel[];
spotlight: SpotlightTileViewModel;
}
/**
@@ -161,6 +209,12 @@ enum SortingBin {
SelfNotAlwaysShown,
}
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
@@ -426,12 +480,6 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.screenShares.pipe(
map((ms) => ms.some((m) => !m.vm.local)),
distinctUntilChanged(),
);
private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
this.userMedia.pipe(
switchMap((mediaItems) =>
@@ -466,7 +514,6 @@ export class CallViewModel extends ViewModel {
),
map((speaker) => speaker.vm),
this.scope.state(),
throttleTime(1600, undefined, { leading: true, trailing: true }),
);
private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
@@ -536,6 +583,14 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
@@ -616,7 +671,7 @@ export class CallViewModel extends ViewModel {
screenShares.length === 0,
);
private readonly gridLayout: Observable<Layout> = combineLatest(
private readonly gridLayout: Observable<LayoutMedia> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({
type: "grid",
@@ -627,38 +682,44 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightLandscapeLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }),
);
private readonly spotlightLandscapeLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-landscape",
spotlight,
grid,
}));
private readonly spotlightPortraitLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }),
);
private readonly spotlightPortraitLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-portrait",
spotlight,
grid,
}));
private readonly spotlightExpandedLayout: Observable<Layout> = combineLatest(
[this.spotlight, this.pip],
(spotlight, pip) => ({
private readonly spotlightExpandedLayout: Observable<LayoutMedia> =
combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}),
);
}));
private readonly oneOnOneLayout: Observable<Layout> = this.grid.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.local) as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel,
})),
);
private readonly oneOnOneLayout: Observable<LayoutMedia> =
this.mediaItems.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel,
})),
);
private readonly pipLayout: Observable<Layout> = this.spotlight.pipe(
private readonly pipLayout: Observable<LayoutMedia> = this.spotlight.pipe(
map((spotlight) => ({ type: "pip", spotlight })),
);
public readonly layout: Observable<Layout> = this.windowMode.pipe(
/**
* The media to be used to produce a layout.
*/
private readonly layoutMedia: Observable<LayoutMedia> = this.windowMode.pipe(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
@@ -719,48 +780,95 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
/**
* The layout of tiles in the call interface.
*/
public readonly layout: Observable<Layout> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
>(
({ tiles: prevTiles, visibleTiles }, media) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
media,
visibleTiles,
prevTiles,
);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
},
),
map(({ layout }) => layout),
this.scope.state(),
);
public showSpotlightIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "grid"),
this.scope.state(),
);
/**
* Determines whether video should be shown for a certain piece of media
* appearing in the grid.
*/
public showGridVideo(vm: MediaViewModel): Observable<boolean> {
return this.layout.pipe(
map(
(l) =>
!(
(l.type === "spotlight-landscape" ||
l.type === "spotlight-portrait") &&
// This media is already visible in the spotlight; avoid duplication
l.spotlight.some((spotlightVm) => spotlightVm === vm)
),
),
distinctUntilChanged(),
);
}
public showSpeakingIndicators: Observable<boolean> = this.layout.pipe(
map((l) => {
switchMap((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight[0] instanceof ScreenShareViewModel;
return l.spotlight.media.pipe(
map((models: MediaViewModel[]) =>
models.some((m) => m instanceof ScreenShareViewModel),
),
);
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one":
return false;
return of(false);
default:
return true;
return of(true);
}
}),
this.scope.state(),