🏗️(frontend) decouple landmark processor from background processors

Separate landmark processor logic to avoid entanglement with background
processing. Ensures future refactoring can replace custom background
implementation without affecting landmark functionality.

src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts

@@ -1,4 +1,4 @@
-import { ProcessorOptions, Track } from 'livekit-client'
+import { ProcessorOptions, Track, TrackProcessor } from 'livekit-client'
 import posthog from 'posthog-js'
 import {
   FilesetResolver,
@@ -11,19 +11,20 @@ import {
   TIMEOUT_TICK,
   timerWorkerScript,
 } from './TimerWorker'
-import {
-  BackgroundProcessorInterface,
-  BackgroundOptions,
-  ProcessorType,
-} from '.'
+import { ProcessorType } from '.'
 
 const PROCESSING_WIDTH = 256 * 3
 const PROCESSING_HEIGHT = 144 * 3
 
 const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local'
 
-export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
-  options: BackgroundOptions
+export type FaceLandmarksOptions = {
+  showGlasses: boolean
+  showFrench: boolean
+}
+
+export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
+  options: FaceLandmarksOptions
   name: string
   processedTrack?: MediaStreamTrack | undefined
 
@@ -50,7 +51,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
   glassesImage?: HTMLImageElement
   mustacheImage?: HTMLImageElement
   beretImage?: HTMLImageElement
-  constructor(opts: BackgroundOptions) {
+  constructor(opts: FaceLandmarksOptions) {
     this.name = 'face_landmarks'
     this.options = opts
     this.type = ProcessorType.FACE_LANDMARKS
@@ -314,7 +315,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
     return element
   }
 
-  update(opts: BackgroundOptions): void {
+  update(opts: FaceLandmarksOptions): void {
     this.options = opts
   }
 

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -3,13 +3,10 @@ import { Track, TrackProcessor } from 'livekit-client'
 import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
 import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
 import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'
-import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
 
 export type BackgroundOptions = {
   blurRadius?: number
   imagePath?: string
-  showGlasses?: boolean
-  showFrench?: boolean
 }
 
 export interface ProcessorSerialized {
@@ -37,11 +34,7 @@ export class BackgroundProcessorFactory {
   }
 
   static isSupported() {
-    return (
-      ProcessorWrapper.isSupported ||
-      BackgroundCustomProcessor.isSupported ||
-      FaceLandmarksProcessor.isSupported
-    )
+    return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported
   }
 
   static getProcessor(
@@ -62,10 +55,6 @@ export class BackgroundProcessorFactory {
       if (BackgroundCustomProcessor.isSupported) {
         return new BackgroundCustomProcessor(opts)
       }
-    } else if (type === ProcessorType.FACE_LANDMARKS) {
-      if (FaceLandmarksProcessor.isSupported) {
-        return new FaceLandmarksProcessor(opts)
-      }
     }
     return undefined
   }

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -15,11 +15,8 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
 import { useTrackToggle } from '@livekit/components-react'
 import { Loader } from '@/primitives/Loader'
 import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import {
-  RiProhibited2Line,
-  RiGlassesLine,
-  RiGoblet2Fill,
-} from '@remixicon/react'
+import { RiProhibited2Line } from '@remixicon/react'
+import { FunnyEffects } from './FunnyEffects'
 import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'
 
 enum BlurRadius {
@@ -152,42 +149,9 @@ export const EffectsConfiguration = ({
   }
 
   const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {
-    if (type === ProcessorType.FACE_LANDMARKS) {
-      const effect = options.showGlasses ? 'glasses' : 'french'
-      return t(
-        `faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`
-      )
-    }
     return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)
   }
 
-  const getFaceLandmarksOptions = () => {
-    const processor = getProcessor()
-    if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {
-      return processor.serialize().options as {
-        showGlasses?: boolean
-        showFrench?: boolean
-      }
-    }
-    return { showGlasses: false, showFrench: false }
-  }
-
-  const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'french') => {
-    const currentOptions = getFaceLandmarksOptions()
-    const newOptions = {
-      ...currentOptions,
-      [effect === 'glasses' ? 'showGlasses' : 'showFrench']:
-        !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showFrench'],
-    }
-
-    if (!newOptions.showGlasses && !newOptions.showFrench) {
-      // If both effects are off stop the processor
-      await clearEffect()
-    } else {
-      await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)
-    }
-  }
-
   return (
     <div
       className={css(
@@ -275,6 +239,13 @@ export const EffectsConfiguration = ({
             : {}
         )}
       >
+        {hasFaceLandmarksAccess && (
+          <FunnyEffects
+            videoTrack={videoTrack}
+            isPending={processorPendingReveal}
+            onPending={setProcessorPending}
+          />
+        )}
         {isSupported ? (
           <>
             <div>
@@ -347,8 +318,6 @@ export const EffectsConfiguration = ({
                     <BlurOnStrong />
                   </ToggleButton>
                 </div>
-              </div>
-              {hasFaceLandmarksAccess && (
                 <div
                   className={css({
                     marginTop: '1.5rem',
@@ -361,108 +330,48 @@ export const EffectsConfiguration = ({
                     }}
                     variant="bodyXsBold"
                   >
-                    {t('faceLandmarks.title')}
+                    {t('virtual.title')}
                   </H>
                   <div
                     className={css({
                       display: 'flex',
                       gap: '1.25rem',
+                      flexWrap: 'wrap',
                     })}
                   >
-                    <ToggleButton
-                      variant="bigSquare"
-                      aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        showGlasses: true,
-                        showFrench: false,
-                      })}
-                      tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        showGlasses: true,
-                        showFrench: false,
-                      })}
-                      isDisabled={processorPendingReveal}
-                      onChange={async () =>
-                        await toggleFaceLandmarkEffect('glasses')
-                      }
-                      isSelected={getFaceLandmarksOptions().showGlasses}
-                      data-attr="toggle-glasses"
-                    >
-                      <RiGlassesLine />
-                    </ToggleButton>
-                    <ToggleButton
-                      variant="bigSquare"
-                      aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        showGlasses: false,
-                        showFrench: true,
-                      })}
-                      tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        showGlasses: false,
-                        showFrench: true,
-                      })}
-                      isDisabled={processorPendingReveal}
-                      onChange={async () =>
-                        await toggleFaceLandmarkEffect('french')
-                      }
-                      isSelected={getFaceLandmarksOptions().showFrench}
-                      data-attr="toggle-french"
-                    >
-                      <RiGoblet2Fill />
-                    </ToggleButton>
-                  </div>
-                </div>
-              )}
-              <div
-                className={css({
-                  marginTop: '1.5rem',
-                })}
-              >
-                <H
-                  lvl={3}
-                  style={{
-                    marginBottom: '1rem',
-                  }}
-                  variant="bodyXsBold"
-                >
-                  {t('virtual.title')}
-                </H>
-                <div
-                  className={css({
-                    display: 'flex',
-                    gap: '1.25rem',
-                    flexWrap: 'wrap',
-                  })}
-                >
-                  {[...Array(8).keys()].map((i) => {
-                    const imagePath = `/assets/backgrounds/${i + 1}.jpg`
-                    const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`
-                    return (
-                      <ToggleButton
-                        key={i}
-                        variant="bigSquare"
-                        aria-label={tooltipLabel(ProcessorType.VIRTUAL, {
-                          imagePath,
-                        })}
-                        tooltip={tooltipLabel(ProcessorType.VIRTUAL, {
-                          imagePath,
-                        })}
-                        isDisabled={processorPendingReveal}
-                        onChange={async () =>
-                          await toggleEffect(ProcessorType.VIRTUAL, {
+                    {[...Array(8).keys()].map((i) => {
+                      const imagePath = `/assets/backgrounds/${i + 1}.jpg`
+                      const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`
+                      return (
+                        <ToggleButton
+                          key={i}
+                          variant="bigSquare"
+                          aria-label={tooltipLabel(ProcessorType.VIRTUAL, {
                             imagePath,
-                          })
-                        }
-                        isSelected={isSelected(ProcessorType.VIRTUAL, {
-                          imagePath,
-                        })}
-                        className={css({
-                          bgSize: 'cover',
-                        })}
-                        style={{
-                          backgroundImage: `url(${thumbnailPath})`,
-                        }}
-                        data-attr={`toggle-virtual-${i}`}
-                      />
-                    )
-                  })}
+                          })}
+                          tooltip={tooltipLabel(ProcessorType.VIRTUAL, {
+                            imagePath,
+                          })}
+                          isDisabled={processorPendingReveal}
+                          onChange={async () =>
+                            await toggleEffect(ProcessorType.VIRTUAL, {
+                              imagePath,
+                            })
+                          }
+                          isSelected={isSelected(ProcessorType.VIRTUAL, {
+                            imagePath,
+                          })}
+                          className={css({
+                            bgSize: 'cover',
+                          })}
+                          style={{
+                            backgroundImage: `url(${thumbnailPath})`,
+                          }}
+                          data-attr={`toggle-virtual-${i}`}
+                        />
+                      )
+                    })}
+                  </div>
                 </div>
               </div>
             </div>

src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx

@@ -0,0 +1,120 @@
+import { css } from '@/styled-system/css'
+import { H, ToggleButton } from '@/primitives'
+import { ProcessorType } from '../blur'
+import { RiGlassesLine, RiGoblet2Fill } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import { FaceLandmarksProcessor } from '../blur/FaceLandmarksProcessor'
+import { LocalVideoTrack } from 'livekit-client'
+
+export type FunnyEffectsProps = {
+  videoTrack: LocalVideoTrack
+  isPending?: boolean
+  onPending: (value: boolean) => void
+}
+
+export const FunnyEffects = ({
+  videoTrack,
+  isPending,
+  onPending,
+}: FunnyEffectsProps) => {
+  const { t } = useTranslation('rooms', { keyPrefix: 'effects' })
+
+  const getOptions = () => {
+    const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor
+    if (!processor || processor.type != ProcessorType.FACE_LANDMARKS) {
+      return {
+        showGlasses: false,
+        showFrench: false,
+      }
+    }
+    return processor.serialize().options
+  }
+
+  const options = getOptions()
+
+  const toggleFaceLandmarkEffect = async (
+    showEffect: 'showGlasses' | 'showFrench'
+  ) => {
+    const options = getOptions()
+    const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor
+
+    const newOptions = {
+      ...options,
+      [showEffect]: !options[showEffect],
+    }
+
+    onPending(true)
+
+    try {
+      if (!newOptions.showGlasses && !newOptions.showFrench) {
+        await videoTrack.stopProcessor()
+      } else if (options.showGlasses || options.showFrench) {
+        await processor?.update(newOptions)
+      } else {
+        const newProcessor = new FaceLandmarksProcessor(newOptions)
+        await videoTrack.setProcessor(newProcessor)
+      }
+    } catch (e) {
+      console.error('could not update processor', e)
+    } finally {
+      onPending(false)
+    }
+  }
+
+  const getLabelAction = (enabled: boolean) => (enabled ? 'clear' : 'apply')
+
+  return (
+    <div
+      className={css({
+        marginBottom: '1.5rem',
+      })}
+    >
+      <H
+        lvl={3}
+        style={{
+          marginBottom: '1rem',
+        }}
+        variant="bodyXsBold"
+      >
+        {t('faceLandmarks.title')}
+      </H>
+      <div
+        className={css({
+          display: 'flex',
+          gap: '1.25rem',
+        })}
+      >
+        <ToggleButton
+          variant="bigSquare"
+          aria-label={t(
+            `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`
+          )}
+          tooltip={t(
+            `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`
+          )}
+          isDisabled={isPending}
+          onChange={async () => await toggleFaceLandmarkEffect('showGlasses')}
+          isSelected={options.showGlasses}
+          data-attr="toggle-glasses"
+        >
+          <RiGlassesLine />
+        </ToggleButton>
+        <ToggleButton
+          variant="bigSquare"
+          aria-label={t(
+            `faceLandmarks.french.${getLabelAction(options.showFrench)}`
+          )}
+          tooltip={t(
+            `faceLandmarks.french.${getLabelAction(options.showFrench)}`
+          )}
+          isDisabled={isPending}
+          onChange={async () => await toggleFaceLandmarkEffect('showFrench')}
+          isSelected={options.showFrench}
+          data-attr="toggle-french"
+        >
+          <RiGoblet2Fill />
+        </ToggleButton>
+      </div>
+    </div>
+  )
+}