diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx index 57c28c44..c26e4465 100644 --- a/src/frontend/src/features/rooms/components/Conference.tsx +++ b/src/frontend/src/features/rooms/components/Conference.tsx @@ -16,7 +16,7 @@ import { VideoConference } from '../livekit/prefabs/VideoConference' import posthog from 'posthog-js' import { css } from '@/styled-system/css' import { LocalUserChoices } from '../routes/Room' -import { BackgroundBlurFactory } from '../livekit/components/blur' +import { BackgroundProcessorFactory } from '../livekit/components/blur' export const Conference = ({ roomId, @@ -114,7 +114,7 @@ export const Conference = ({ audio={userConfig.audioEnabled} video={ userConfig.videoEnabled && { - processor: BackgroundBlurFactory.deserializeProcessor( + processor: BackgroundProcessorFactory.deserializeProcessor( userConfig.processorSerialized ), } diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsx index 4a942545..61cbfd39 100644 --- a/src/frontend/src/features/rooms/components/Join.tsx +++ b/src/frontend/src/features/rooms/components/Join.tsx @@ -7,9 +7,8 @@ import { LocalVideoTrack, Track } from 'livekit-client' import { H } from '@/primitives/H' import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice' import { Field } from '@/primitives/Field' -import { Form } from '@/primitives' +import { Button, Dialog, Text, Form } from '@/primitives' import { HStack, VStack } from '@/styled-system/jsx' -import { Button, Dialog } from '@/primitives' import { LocalUserChoices } from '../routes/Room' import { Heading } from 'react-aria-components' import { RiImageCircleAiFill } from '@remixicon/react' @@ -18,7 +17,7 @@ import { EffectsConfigurationProps, } from '../livekit/components/effects/EffectsConfiguration' import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices' -import { BackgroundBlurFactory } from '../livekit/components/blur' +import { BackgroundProcessorFactory } from '../livekit/components/blur' import { isMobileBrowser } from '@livekit/components-core' const onError = (e: Error) => console.error('ERROR', e) @@ -31,7 +30,7 @@ const Effects = ({ const [isDialogOpen, setIsDialogOpen] = useState(false) const openDialog = () => setIsDialogOpen(true) - if (!BackgroundBlurFactory.isSupported() || isMobileBrowser()) { + if (!BackgroundProcessorFactory.isSupported() || isMobileBrowser()) { return } @@ -49,11 +48,19 @@ const Effects = ({ level={1} className={css({ textStyle: 'h1', - marginBottom: '1.5rem', + marginBottom: '0.25rem', })} > {t('title')} + + {t('subTitle')} +
( initialUserChoices.audioDeviceId ) @@ -113,7 +122,7 @@ export const Join = ({ ) const [username, setUsername] = useState(initialUserChoices.username) const [processor, setProcessor] = useState( - BackgroundBlurFactory.deserializeProcessor( + BackgroundProcessorFactory.deserializeProcessor( initialUserChoices.processorSerialized ) ) @@ -129,12 +138,15 @@ export const Join = ({ useEffect(() => { saveUsername(username) }, [username, saveUsername]) + useEffect(() => { saveProcessorSerialized(processor?.serialize()) - }, [processor, saveProcessorSerialized]) - - const [audioEnabled, setAudioEnabled] = useState(true) - const [videoEnabled, setVideoEnabled] = useState(true) + }, [ + processor, + saveProcessorSerialized, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify(processor?.serialize()), + ]) const tracks = usePreviewTracks( { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts index 5481a9ff..9d362c7f 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts @@ -5,7 +5,7 @@ import { } from '@livekit/track-processors' import { ProcessorOptions, Track } from 'livekit-client' import { - BackgroundBlurProcessorInterface, + BackgroundProcessorInterface, BackgroundOptions, ProcessorType, } from '.' @@ -16,9 +16,9 @@ import { * used accross the project. */ export class BackgroundBlurTrackProcessorJsWrapper - implements BackgroundBlurProcessorInterface + implements BackgroundProcessorInterface { - name: string = 'Blur' + name: string = 'blur' processor: ProcessorWrapper diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts similarity index 76% rename from src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts rename to src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 781e0d66..744a749f 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -12,7 +12,7 @@ import { timerWorkerScript, } from './TimerWorker' import { - BackgroundBlurProcessorInterface, + BackgroundProcessorInterface, BackgroundOptions, ProcessorType, } from '.' @@ -32,9 +32,7 @@ const DEFAULT_BLUR = '10' * It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and * MediaStreamTrackProcessor. */ -export class BackgroundBlurCustomProcessor - implements BackgroundBlurProcessorInterface -{ +export class BackgroundCustomProcessor implements BackgroundProcessorInterface { options: BackgroundOptions name: string processedTrack?: MediaStreamTrack | undefined @@ -63,9 +61,18 @@ export class BackgroundBlurCustomProcessor timerWorker?: Worker + type: ProcessorType + virtualBackgroundImage?: HTMLImageElement + constructor(opts: BackgroundOptions) { this.name = 'blur' this.options = opts + + if (this.options.blurRadius) { + this.type = ProcessorType.BLUR + } else { + this.type = ProcessorType.VIRTUAL + } } static get isSupported() { @@ -81,6 +88,7 @@ export class BackgroundBlurCustomProcessor this.sourceSettings = this.source!.getSettings() this.videoElement = opts.element as HTMLVideoElement + this._initVirtualBackgroundImage() this._createMainCanvas() this._createMaskCanvas() @@ -98,8 +106,21 @@ export class BackgroundBlurCustomProcessor posthog.capture('firefox-blurring-init') } + _initVirtualBackgroundImage() { + const needsUpdate = + this.options.imagePath && + this.virtualBackgroundImage && + this.virtualBackgroundImage.src !== this.options.imagePath + if (this.options.imagePath || needsUpdate) { + this.virtualBackgroundImage = document.createElement('img') + this.virtualBackgroundImage.crossOrigin = 'anonymous' + this.virtualBackgroundImage.src = this.options.imagePath! + } + } + update(opts: BackgroundOptions): void { this.options = opts + this._initVirtualBackgroundImage() } _initWorker() { @@ -201,6 +222,7 @@ export class BackgroundBlurCustomProcessor this.outputCanvasCtx!.globalCompositeOperation = 'copy' this.outputCanvasCtx!.filter = 'blur(8px)' + // Put opacity mask. this.outputCanvasCtx!.drawImage( this.segmentationMaskCanvas!, 0, @@ -213,19 +235,69 @@ export class BackgroundBlurCustomProcessor this.videoElement!.videoHeight ) + // Draw clear body. this.outputCanvasCtx!.globalCompositeOperation = 'source-in' this.outputCanvasCtx!.filter = 'none' this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + // Draw blurry background. this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)` this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) } + /** + * TODO: future improvement with WebGL. + */ + async drawVirtualBackground() { + const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() + for (let i = 0; i < mask.length; ++i) { + this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] + } + + this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) + + this.outputCanvasCtx!.globalCompositeOperation = 'copy' + this.outputCanvasCtx!.filter = 'blur(8px)' + + // Put opacity mask. + this.outputCanvasCtx!.drawImage( + this.segmentationMaskCanvas!, + 0, + 0, + PROCESSING_WIDTH, + PROCESSING_HEIGHT, + 0, + 0, + this.videoElement!.videoWidth, + this.videoElement!.videoHeight + ) + + // Draw clear body. + this.outputCanvasCtx!.globalCompositeOperation = 'source-in' + this.outputCanvasCtx!.filter = 'none' + this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + + // Draw virtual background. + this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' + this.outputCanvasCtx!.drawImage( + this.virtualBackgroundImage!, + 0, + 0, + this.outputCanvas!.width, + this.outputCanvas!.height + ) + } + async process() { await this.sizeSource() await this.segment() - await this.blur() + + if (this.options.blurRadius) { + await this.blur() + } else { + await this.drawVirtualBackground() + } this.timerWorker!.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30, @@ -284,12 +356,12 @@ export class BackgroundBlurCustomProcessor } clone() { - return new BackgroundBlurCustomProcessor(this.options) + return new BackgroundCustomProcessor(this.options) } serialize() { return { - type: ProcessorType.BLUR, + type: this.type, options: this.options, } } diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index eaa4a372..893b9eb0 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -1,10 +1,12 @@ import { ProcessorWrapper } from '@livekit/track-processors' import { Track, TrackProcessor } from 'livekit-client' import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper' -import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor' +import { BackgroundCustomProcessor } from './BackgroundCustomProcessor' +import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper' export type BackgroundOptions = { blurRadius?: number + imagePath?: string } export interface ProcessorSerialized { @@ -12,40 +14,49 @@ export interface ProcessorSerialized { options: BackgroundOptions } -export interface BackgroundBlurProcessorInterface +export interface BackgroundProcessorInterface extends TrackProcessor { update(opts: BackgroundOptions): void options: BackgroundOptions - clone(): BackgroundBlurProcessorInterface + clone(): BackgroundProcessorInterface serialize(): ProcessorSerialized } export enum ProcessorType { BLUR = 'blur', + VIRTUAL = 'virtual', } -export class BackgroundBlurFactory { +export class BackgroundProcessorFactory { static isSupported() { - return ( - ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported - ) + return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported } static getProcessor( + type: ProcessorType, opts: BackgroundOptions - ): BackgroundBlurProcessorInterface | undefined { - if (ProcessorWrapper.isSupported) { - return new BackgroundBlurTrackProcessorJsWrapper(opts) - } - if (BackgroundBlurCustomProcessor.isSupported) { - return new BackgroundBlurCustomProcessor(opts) + ): BackgroundProcessorInterface | undefined { + if (type === ProcessorType.BLUR) { + if (ProcessorWrapper.isSupported) { + return new BackgroundBlurTrackProcessorJsWrapper(opts) + } + if (BackgroundCustomProcessor.isSupported) { + return new BackgroundCustomProcessor(opts) + } + } else if (type === ProcessorType.VIRTUAL) { + if (ProcessorWrapper.isSupported) { + return new BackgroundVirtualTrackProcessorJsWrapper(opts) + } + if (BackgroundCustomProcessor.isSupported) { + return new BackgroundCustomProcessor(opts) + } } return undefined } static deserializeProcessor(data?: ProcessorSerialized) { - if (data?.type === ProcessorType.BLUR) { - return BackgroundBlurFactory.getProcessor(data?.options) + if (data?.type) { + return BackgroundProcessorFactory.getProcessor(data?.type, data?.options) } return undefined } diff --git a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx index efb6a91a..46dd442c 100644 --- a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx +++ b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx @@ -27,7 +27,7 @@ import { css } from '@/styled-system/css' import { ButtonRecipeProps } from '@/primitives/buttonRecipe' import { useEffect } from 'react' import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices' -import { BackgroundBlurFactory } from '../blur' +import { BackgroundProcessorFactory } from '../blur' export type ToggleSource = Exclude< Track.Source, @@ -115,7 +115,7 @@ export const SelectToggleDevice = ({ * * See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121 */ - const processor = BackgroundBlurFactory.deserializeProcessor( + const processor = BackgroundProcessorFactory.deserializeProcessor( userChoices.processorSerialized ) diff --git a/src/frontend/src/primitives/Text.tsx b/src/frontend/src/primitives/Text.tsx index 37c2d576..15231f2a 100644 --- a/src/frontend/src/primitives/Text.tsx +++ b/src/frontend/src/primitives/Text.tsx @@ -28,6 +28,14 @@ export const text = cva({ paddingTop: 'heading', }, }, + subTitle: { + fontSize: '1rem', + color: 'greyscale.600', + }, + bodyXsBold: { + textStyle: 'body', + fontWeight: 'bold', + }, body: { textStyle: 'body', },