✨(front) handle virtual background in custom processor

BackgroundBlurCustomProcessor is renamed to BackgroundCustomProcessor in
order to reflect the fact that is now handles virtual backgrounds too.
BackgroundBlurFactory is also renamed to BackgroundProcessorFactory.
The processor serialization handling has also been updated in order
to support various options, also if persisted in local storage.

src/frontend/src/features/rooms/components/Conference.tsx

@@ -16,7 +16,7 @@ import { VideoConference } from '../livekit/prefabs/VideoConference'
 import posthog from 'posthog-js'
 import { css } from '@/styled-system/css'
 import { LocalUserChoices } from '../routes/Room'
-import { BackgroundBlurFactory } from '../livekit/components/blur'
+import { BackgroundProcessorFactory } from '../livekit/components/blur'
 
 export const Conference = ({
   roomId,
@@ -114,7 +114,7 @@ export const Conference = ({
           audio={userConfig.audioEnabled}
           video={
             userConfig.videoEnabled && {
-              processor: BackgroundBlurFactory.deserializeProcessor(
+              processor: BackgroundProcessorFactory.deserializeProcessor(
                 userConfig.processorSerialized
               ),
             }

src/frontend/src/features/rooms/components/Join.tsx

@@ -7,9 +7,8 @@ import { LocalVideoTrack, Track } from 'livekit-client'
 import { H } from '@/primitives/H'
 import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice'
 import { Field } from '@/primitives/Field'
-import { Form } from '@/primitives'
+import { Button, Dialog, Text, Form } from '@/primitives'
 import { HStack, VStack } from '@/styled-system/jsx'
-import { Button, Dialog } from '@/primitives'
 import { LocalUserChoices } from '../routes/Room'
 import { Heading } from 'react-aria-components'
 import { RiImageCircleAiFill } from '@remixicon/react'
@@ -18,7 +17,7 @@ import {
   EffectsConfigurationProps,
 } from '../livekit/components/effects/EffectsConfiguration'
 import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
-import { BackgroundBlurFactory } from '../livekit/components/blur'
+import { BackgroundProcessorFactory } from '../livekit/components/blur'
 import { isMobileBrowser } from '@livekit/components-core'
 
 const onError = (e: Error) => console.error('ERROR', e)
@@ -31,7 +30,7 @@ const Effects = ({
   const [isDialogOpen, setIsDialogOpen] = useState(false)
   const openDialog = () => setIsDialogOpen(true)
 
-  if (!BackgroundBlurFactory.isSupported() || isMobileBrowser()) {
+  if (!BackgroundProcessorFactory.isSupported() || isMobileBrowser()) {
     return
   }
 
@@ -49,11 +48,19 @@ const Effects = ({
           level={1}
           className={css({
             textStyle: 'h1',
-            marginBottom: '1.5rem',
+            marginBottom: '0.25rem',
           })}
         >
           {t('title')}
         </Heading>
+        <Text
+          variant="subTitle"
+          className={css({
+            marginBottom: '1.5rem',
+          })}
+        >
+          {t('subTitle')}
+        </Text>
         <EffectsConfiguration videoTrack={videoTrack} onSubmit={onSubmit} />
       </Dialog>
       <div
@@ -105,6 +112,8 @@ export const Join = ({
     saveProcessorSerialized,
   } = usePersistentUserChoices({})
 
+  const [audioEnabled, setAudioEnabled] = useState(true)
+  const [videoEnabled, setVideoEnabled] = useState(true)
   const [audioDeviceId, setAudioDeviceId] = useState<string>(
     initialUserChoices.audioDeviceId
   )
@@ -113,7 +122,7 @@ export const Join = ({
   )
   const [username, setUsername] = useState<string>(initialUserChoices.username)
   const [processor, setProcessor] = useState(
-    BackgroundBlurFactory.deserializeProcessor(
+    BackgroundProcessorFactory.deserializeProcessor(
       initialUserChoices.processorSerialized
     )
   )
@@ -129,12 +138,15 @@ export const Join = ({
   useEffect(() => {
     saveUsername(username)
   }, [username, saveUsername])
+
   useEffect(() => {
     saveProcessorSerialized(processor?.serialize())
-  }, [processor, saveProcessorSerialized])
-
-  const [audioEnabled, setAudioEnabled] = useState(true)
-  const [videoEnabled, setVideoEnabled] = useState(true)
+  }, [
+    processor,
+    saveProcessorSerialized,
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    JSON.stringify(processor?.serialize()),
+  ])
 
   const tracks = usePreviewTracks(
     {

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts

@@ -5,7 +5,7 @@ import {
 } from '@livekit/track-processors'
 import { ProcessorOptions, Track } from 'livekit-client'
 import {
-  BackgroundBlurProcessorInterface,
+  BackgroundProcessorInterface,
   BackgroundOptions,
   ProcessorType,
 } from '.'
@@ -16,9 +16,9 @@ import {
  * used accross the project.
  */
 export class BackgroundBlurTrackProcessorJsWrapper
-  implements BackgroundBlurProcessorInterface
+  implements BackgroundProcessorInterface
 {
-  name: string = 'Blur'
+  name: string = 'blur'
 
   processor: ProcessorWrapper<BackgroundOptions>
 

src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts

@@ -12,7 +12,7 @@ import {
   timerWorkerScript,
 } from './TimerWorker'
 import {
-  BackgroundBlurProcessorInterface,
+  BackgroundProcessorInterface,
   BackgroundOptions,
   ProcessorType,
 } from '.'
@@ -32,9 +32,7 @@ const DEFAULT_BLUR = '10'
  * It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and
  * MediaStreamTrackProcessor.
  */
-export class BackgroundBlurCustomProcessor
-  implements BackgroundBlurProcessorInterface
-{
+export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   options: BackgroundOptions
   name: string
   processedTrack?: MediaStreamTrack | undefined
@@ -63,9 +61,18 @@ export class BackgroundBlurCustomProcessor
 
   timerWorker?: Worker
 
+  type: ProcessorType
+  virtualBackgroundImage?: HTMLImageElement
+
   constructor(opts: BackgroundOptions) {
     this.name = 'blur'
     this.options = opts
+
+    if (this.options.blurRadius) {
+      this.type = ProcessorType.BLUR
+    } else {
+      this.type = ProcessorType.VIRTUAL
+    }
   }
 
   static get isSupported() {
@@ -81,6 +88,7 @@ export class BackgroundBlurCustomProcessor
     this.sourceSettings = this.source!.getSettings()
     this.videoElement = opts.element as HTMLVideoElement
 
+    this._initVirtualBackgroundImage()
     this._createMainCanvas()
     this._createMaskCanvas()
 
@@ -98,8 +106,21 @@ export class BackgroundBlurCustomProcessor
     posthog.capture('firefox-blurring-init')
   }
 
+  _initVirtualBackgroundImage() {
+    const needsUpdate =
+      this.options.imagePath &&
+      this.virtualBackgroundImage &&
+      this.virtualBackgroundImage.src !== this.options.imagePath
+    if (this.options.imagePath || needsUpdate) {
+      this.virtualBackgroundImage = document.createElement('img')
+      this.virtualBackgroundImage.crossOrigin = 'anonymous'
+      this.virtualBackgroundImage.src = this.options.imagePath!
+    }
+  }
+
   update(opts: BackgroundOptions): void {
     this.options = opts
+    this._initVirtualBackgroundImage()
   }
 
   _initWorker() {
@@ -201,6 +222,7 @@ export class BackgroundBlurCustomProcessor
     this.outputCanvasCtx!.globalCompositeOperation = 'copy'
     this.outputCanvasCtx!.filter = 'blur(8px)'
 
+    // Put opacity mask.
     this.outputCanvasCtx!.drawImage(
       this.segmentationMaskCanvas!,
       0,
@@ -213,19 +235,69 @@ export class BackgroundBlurCustomProcessor
       this.videoElement!.videoHeight
     )
 
+    // Draw clear body.
     this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
     this.outputCanvasCtx!.filter = 'none'
     this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
 
+    // Draw blurry background.
     this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
     this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)`
     this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
   }
 
+  /**
+   * TODO: future improvement with WebGL.
+   */
+  async drawVirtualBackground() {
+    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
+    for (let i = 0; i < mask.length; ++i) {
+      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
+    }
+
+    this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
+
+    this.outputCanvasCtx!.globalCompositeOperation = 'copy'
+    this.outputCanvasCtx!.filter = 'blur(8px)'
+
+    // Put opacity mask.
+    this.outputCanvasCtx!.drawImage(
+      this.segmentationMaskCanvas!,
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT,
+      0,
+      0,
+      this.videoElement!.videoWidth,
+      this.videoElement!.videoHeight
+    )
+
+    // Draw clear body.
+    this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
+    this.outputCanvasCtx!.filter = 'none'
+    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+
+    // Draw virtual background.
+    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
+    this.outputCanvasCtx!.drawImage(
+      this.virtualBackgroundImage!,
+      0,
+      0,
+      this.outputCanvas!.width,
+      this.outputCanvas!.height
+    )
+  }
+
   async process() {
     await this.sizeSource()
     await this.segment()
-    await this.blur()
+
+    if (this.options.blurRadius) {
+      await this.blur()
+    } else {
+      await this.drawVirtualBackground()
+    }
     this.timerWorker!.postMessage({
       id: SET_TIMEOUT,
       timeMs: 1000 / 30,
@@ -284,12 +356,12 @@ export class BackgroundBlurCustomProcessor
   }
 
   clone() {
-    return new BackgroundBlurCustomProcessor(this.options)
+    return new BackgroundCustomProcessor(this.options)
   }
 
   serialize() {
     return {
-      type: ProcessorType.BLUR,
+      type: this.type,
       options: this.options,
     }
   }

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -1,10 +1,12 @@
 import { ProcessorWrapper } from '@livekit/track-processors'
 import { Track, TrackProcessor } from 'livekit-client'
 import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
-import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor'
+import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
+import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'
 
 export type BackgroundOptions = {
   blurRadius?: number
+  imagePath?: string
 }
 
 export interface ProcessorSerialized {
@@ -12,40 +14,49 @@ export interface ProcessorSerialized {
   options: BackgroundOptions
 }
 
-export interface BackgroundBlurProcessorInterface
+export interface BackgroundProcessorInterface
   extends TrackProcessor<Track.Kind> {
   update(opts: BackgroundOptions): void
   options: BackgroundOptions
-  clone(): BackgroundBlurProcessorInterface
+  clone(): BackgroundProcessorInterface
   serialize(): ProcessorSerialized
 }
 
 export enum ProcessorType {
   BLUR = 'blur',
+  VIRTUAL = 'virtual',
 }
 
-export class BackgroundBlurFactory {
+export class BackgroundProcessorFactory {
   static isSupported() {
-    return (
-      ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported
-    )
+    return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported
   }
 
   static getProcessor(
+    type: ProcessorType,
     opts: BackgroundOptions
-  ): BackgroundBlurProcessorInterface | undefined {
-    if (ProcessorWrapper.isSupported) {
-      return new BackgroundBlurTrackProcessorJsWrapper(opts)
-    }
-    if (BackgroundBlurCustomProcessor.isSupported) {
-      return new BackgroundBlurCustomProcessor(opts)
+  ): BackgroundProcessorInterface | undefined {
+    if (type === ProcessorType.BLUR) {
+      if (ProcessorWrapper.isSupported) {
+        return new BackgroundBlurTrackProcessorJsWrapper(opts)
+      }
+      if (BackgroundCustomProcessor.isSupported) {
+        return new BackgroundCustomProcessor(opts)
+      }
+    } else if (type === ProcessorType.VIRTUAL) {
+      if (ProcessorWrapper.isSupported) {
+        return new BackgroundVirtualTrackProcessorJsWrapper(opts)
+      }
+      if (BackgroundCustomProcessor.isSupported) {
+        return new BackgroundCustomProcessor(opts)
+      }
     }
     return undefined
   }
 
   static deserializeProcessor(data?: ProcessorSerialized) {
-    if (data?.type === ProcessorType.BLUR) {
-      return BackgroundBlurFactory.getProcessor(data?.options)
+    if (data?.type) {
+      return BackgroundProcessorFactory.getProcessor(data?.type, data?.options)
     }
     return undefined
   }

src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx

@@ -27,7 +27,7 @@ import { css } from '@/styled-system/css'
 import { ButtonRecipeProps } from '@/primitives/buttonRecipe'
 import { useEffect } from 'react'
 import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
-import { BackgroundBlurFactory } from '../blur'
+import { BackgroundProcessorFactory } from '../blur'
 
 export type ToggleSource = Exclude<
   Track.Source,
@@ -115,7 +115,7 @@ export const SelectToggleDevice = <T extends ToggleSource>({
        *
        * See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121
        */
-      const processor = BackgroundBlurFactory.deserializeProcessor(
+      const processor = BackgroundProcessorFactory.deserializeProcessor(
         userChoices.processorSerialized
       )
 

src/frontend/src/primitives/Text.tsx

@@ -28,6 +28,14 @@ export const text = cva({
           paddingTop: 'heading',
         },
       },
+      subTitle: {
+        fontSize: '1rem',
+        color: 'greyscale.600',
+      },
+      bodyXsBold: {
+        textStyle: 'body',
+        fontWeight: 'bold',
+      },
       body: {
         textStyle: 'body',
       },