From 08c5245b48f1af4a949686491f7d7917ab485939 Mon Sep 17 00:00:00 2001 From: Nathan Vasse Date: Fri, 31 Jan 2025 15:42:57 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(front)=20handle=20virtual=20backgroun?= =?UTF-8?q?d=20in=20custom=20processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BackgroundBlurCustomProcessor is renamed to BackgroundCustomProcessor in order to reflect the fact that is now handles virtual backgrounds too. BackgroundBlurFactory is also renamed to BackgroundProcessorFactory. The processor serialization handling has also been updated in order to support various options, also if persisted in local storage. --- .../features/rooms/components/Conference.tsx | 4 +- .../src/features/rooms/components/Join.tsx | 32 ++++--- .../BackgroundBlurTrackProcessorJsWrapper.ts | 6 +- ...cessor.ts => BackgroundCustomProcessor.ts} | 86 +++++++++++++++++-- .../rooms/livekit/components/blur/index.ts | 41 +++++---- .../controls/SelectToggleDevice.tsx | 4 +- src/frontend/src/primitives/Text.tsx | 8 ++ 7 files changed, 142 insertions(+), 39 deletions(-) rename src/frontend/src/features/rooms/livekit/components/blur/{BackgroundBlurCustomProcessor.ts => BackgroundCustomProcessor.ts} (76%) diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx index 57c28c44..c26e4465 100644 --- a/src/frontend/src/features/rooms/components/Conference.tsx +++ b/src/frontend/src/features/rooms/components/Conference.tsx @@ -16,7 +16,7 @@ import { VideoConference } from '../livekit/prefabs/VideoConference' import posthog from 'posthog-js' import { css } from '@/styled-system/css' import { LocalUserChoices } from '../routes/Room' -import { BackgroundBlurFactory } from '../livekit/components/blur' +import { BackgroundProcessorFactory } from '../livekit/components/blur' export const Conference = ({ roomId, @@ -114,7 +114,7 @@ export const Conference = ({ audio={userConfig.audioEnabled} video={ userConfig.videoEnabled && { - processor: BackgroundBlurFactory.deserializeProcessor( + processor: BackgroundProcessorFactory.deserializeProcessor( userConfig.processorSerialized ), } diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsx index 4a942545..61cbfd39 100644 --- a/src/frontend/src/features/rooms/components/Join.tsx +++ b/src/frontend/src/features/rooms/components/Join.tsx @@ -7,9 +7,8 @@ import { LocalVideoTrack, Track } from 'livekit-client' import { H } from '@/primitives/H' import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice' import { Field } from '@/primitives/Field' -import { Form } from '@/primitives' +import { Button, Dialog, Text, Form } from '@/primitives' import { HStack, VStack } from '@/styled-system/jsx' -import { Button, Dialog } from '@/primitives' import { LocalUserChoices } from '../routes/Room' import { Heading } from 'react-aria-components' import { RiImageCircleAiFill } from '@remixicon/react' @@ -18,7 +17,7 @@ import { EffectsConfigurationProps, } from '../livekit/components/effects/EffectsConfiguration' import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices' -import { BackgroundBlurFactory } from '../livekit/components/blur' +import { BackgroundProcessorFactory } from '../livekit/components/blur' import { isMobileBrowser } from '@livekit/components-core' const onError = (e: Error) => console.error('ERROR', e) @@ -31,7 +30,7 @@ const Effects = ({ const [isDialogOpen, setIsDialogOpen] = useState(false) const openDialog = () => setIsDialogOpen(true) - if (!BackgroundBlurFactory.isSupported() || isMobileBrowser()) { + if (!BackgroundProcessorFactory.isSupported() || isMobileBrowser()) { return } @@ -49,11 +48,19 @@ const Effects = ({ level={1} className={css({ textStyle: 'h1', - marginBottom: '1.5rem', + marginBottom: '0.25rem', })} > {t('title')} + + {t('subTitle')} +
( initialUserChoices.audioDeviceId ) @@ -113,7 +122,7 @@ export const Join = ({ ) const [username, setUsername] = useState(initialUserChoices.username) const [processor, setProcessor] = useState( - BackgroundBlurFactory.deserializeProcessor( + BackgroundProcessorFactory.deserializeProcessor( initialUserChoices.processorSerialized ) ) @@ -129,12 +138,15 @@ export const Join = ({ useEffect(() => { saveUsername(username) }, [username, saveUsername]) + useEffect(() => { saveProcessorSerialized(processor?.serialize()) - }, [processor, saveProcessorSerialized]) - - const [audioEnabled, setAudioEnabled] = useState(true) - const [videoEnabled, setVideoEnabled] = useState(true) + }, [ + processor, + saveProcessorSerialized, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify(processor?.serialize()), + ]) const tracks = usePreviewTracks( { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts index 5481a9ff..9d362c7f 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts @@ -5,7 +5,7 @@ import { } from '@livekit/track-processors' import { ProcessorOptions, Track } from 'livekit-client' import { - BackgroundBlurProcessorInterface, + BackgroundProcessorInterface, BackgroundOptions, ProcessorType, } from '.' @@ -16,9 +16,9 @@ import { * used accross the project. */ export class BackgroundBlurTrackProcessorJsWrapper - implements BackgroundBlurProcessorInterface + implements BackgroundProcessorInterface { - name: string = 'Blur' + name: string = 'blur' processor: ProcessorWrapper diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts similarity index 76% rename from src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts rename to src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 781e0d66..744a749f 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -12,7 +12,7 @@ import { timerWorkerScript, } from './TimerWorker' import { - BackgroundBlurProcessorInterface, + BackgroundProcessorInterface, BackgroundOptions, ProcessorType, } from '.' @@ -32,9 +32,7 @@ const DEFAULT_BLUR = '10' * It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and * MediaStreamTrackProcessor. */ -export class BackgroundBlurCustomProcessor - implements BackgroundBlurProcessorInterface -{ +export class BackgroundCustomProcessor implements BackgroundProcessorInterface { options: BackgroundOptions name: string processedTrack?: MediaStreamTrack | undefined @@ -63,9 +61,18 @@ export class BackgroundBlurCustomProcessor timerWorker?: Worker + type: ProcessorType + virtualBackgroundImage?: HTMLImageElement + constructor(opts: BackgroundOptions) { this.name = 'blur' this.options = opts + + if (this.options.blurRadius) { + this.type = ProcessorType.BLUR + } else { + this.type = ProcessorType.VIRTUAL + } } static get isSupported() { @@ -81,6 +88,7 @@ export class BackgroundBlurCustomProcessor this.sourceSettings = this.source!.getSettings() this.videoElement = opts.element as HTMLVideoElement + this._initVirtualBackgroundImage() this._createMainCanvas() this._createMaskCanvas() @@ -98,8 +106,21 @@ export class BackgroundBlurCustomProcessor posthog.capture('firefox-blurring-init') } + _initVirtualBackgroundImage() { + const needsUpdate = + this.options.imagePath && + this.virtualBackgroundImage && + this.virtualBackgroundImage.src !== this.options.imagePath + if (this.options.imagePath || needsUpdate) { + this.virtualBackgroundImage = document.createElement('img') + this.virtualBackgroundImage.crossOrigin = 'anonymous' + this.virtualBackgroundImage.src = this.options.imagePath! + } + } + update(opts: BackgroundOptions): void { this.options = opts + this._initVirtualBackgroundImage() } _initWorker() { @@ -201,6 +222,7 @@ export class BackgroundBlurCustomProcessor this.outputCanvasCtx!.globalCompositeOperation = 'copy' this.outputCanvasCtx!.filter = 'blur(8px)' + // Put opacity mask. this.outputCanvasCtx!.drawImage( this.segmentationMaskCanvas!, 0, @@ -213,19 +235,69 @@ export class BackgroundBlurCustomProcessor this.videoElement!.videoHeight ) + // Draw clear body. this.outputCanvasCtx!.globalCompositeOperation = 'source-in' this.outputCanvasCtx!.filter = 'none' this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + // Draw blurry background. this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)` this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) } + /** + * TODO: future improvement with WebGL. + */ + async drawVirtualBackground() { + const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() + for (let i = 0; i < mask.length; ++i) { + this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] + } + + this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) + + this.outputCanvasCtx!.globalCompositeOperation = 'copy' + this.outputCanvasCtx!.filter = 'blur(8px)' + + // Put opacity mask. + this.outputCanvasCtx!.drawImage( + this.segmentationMaskCanvas!, + 0, + 0, + PROCESSING_WIDTH, + PROCESSING_HEIGHT, + 0, + 0, + this.videoElement!.videoWidth, + this.videoElement!.videoHeight + ) + + // Draw clear body. + this.outputCanvasCtx!.globalCompositeOperation = 'source-in' + this.outputCanvasCtx!.filter = 'none' + this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + + // Draw virtual background. + this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' + this.outputCanvasCtx!.drawImage( + this.virtualBackgroundImage!, + 0, + 0, + this.outputCanvas!.width, + this.outputCanvas!.height + ) + } + async process() { await this.sizeSource() await this.segment() - await this.blur() + + if (this.options.blurRadius) { + await this.blur() + } else { + await this.drawVirtualBackground() + } this.timerWorker!.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30, @@ -284,12 +356,12 @@ export class BackgroundBlurCustomProcessor } clone() { - return new BackgroundBlurCustomProcessor(this.options) + return new BackgroundCustomProcessor(this.options) } serialize() { return { - type: ProcessorType.BLUR, + type: this.type, options: this.options, } } diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index eaa4a372..893b9eb0 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -1,10 +1,12 @@ import { ProcessorWrapper } from '@livekit/track-processors' import { Track, TrackProcessor } from 'livekit-client' import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper' -import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor' +import { BackgroundCustomProcessor } from './BackgroundCustomProcessor' +import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper' export type BackgroundOptions = { blurRadius?: number + imagePath?: string } export interface ProcessorSerialized { @@ -12,40 +14,49 @@ export interface ProcessorSerialized { options: BackgroundOptions } -export interface BackgroundBlurProcessorInterface +export interface BackgroundProcessorInterface extends TrackProcessor { update(opts: BackgroundOptions): void options: BackgroundOptions - clone(): BackgroundBlurProcessorInterface + clone(): BackgroundProcessorInterface serialize(): ProcessorSerialized } export enum ProcessorType { BLUR = 'blur', + VIRTUAL = 'virtual', } -export class BackgroundBlurFactory { +export class BackgroundProcessorFactory { static isSupported() { - return ( - ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported - ) + return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported } static getProcessor( + type: ProcessorType, opts: BackgroundOptions - ): BackgroundBlurProcessorInterface | undefined { - if (ProcessorWrapper.isSupported) { - return new BackgroundBlurTrackProcessorJsWrapper(opts) - } - if (BackgroundBlurCustomProcessor.isSupported) { - return new BackgroundBlurCustomProcessor(opts) + ): BackgroundProcessorInterface | undefined { + if (type === ProcessorType.BLUR) { + if (ProcessorWrapper.isSupported) { + return new BackgroundBlurTrackProcessorJsWrapper(opts) + } + if (BackgroundCustomProcessor.isSupported) { + return new BackgroundCustomProcessor(opts) + } + } else if (type === ProcessorType.VIRTUAL) { + if (ProcessorWrapper.isSupported) { + return new BackgroundVirtualTrackProcessorJsWrapper(opts) + } + if (BackgroundCustomProcessor.isSupported) { + return new BackgroundCustomProcessor(opts) + } } return undefined } static deserializeProcessor(data?: ProcessorSerialized) { - if (data?.type === ProcessorType.BLUR) { - return BackgroundBlurFactory.getProcessor(data?.options) + if (data?.type) { + return BackgroundProcessorFactory.getProcessor(data?.type, data?.options) } return undefined } diff --git a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx index efb6a91a..46dd442c 100644 --- a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx +++ b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx @@ -27,7 +27,7 @@ import { css } from '@/styled-system/css' import { ButtonRecipeProps } from '@/primitives/buttonRecipe' import { useEffect } from 'react' import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices' -import { BackgroundBlurFactory } from '../blur' +import { BackgroundProcessorFactory } from '../blur' export type ToggleSource = Exclude< Track.Source, @@ -115,7 +115,7 @@ export const SelectToggleDevice = ({ * * See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121 */ - const processor = BackgroundBlurFactory.deserializeProcessor( + const processor = BackgroundProcessorFactory.deserializeProcessor( userChoices.processorSerialized ) diff --git a/src/frontend/src/primitives/Text.tsx b/src/frontend/src/primitives/Text.tsx index 37c2d576..15231f2a 100644 --- a/src/frontend/src/primitives/Text.tsx +++ b/src/frontend/src/primitives/Text.tsx @@ -28,6 +28,14 @@ export const text = cva({ paddingTop: 'heading', }, }, + subTitle: { + fontSize: '1rem', + color: 'greyscale.600', + }, + bodyXsBold: { + textStyle: 'body', + fontWeight: 'bold', + }, body: { textStyle: 'body', },