✨(frontend) add face tracking

Implement face landmark processor
to be able to track face live.

src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts

@@ -0,0 +1,251 @@
+import { ProcessorOptions, Track } from 'livekit-client'
+import posthog from 'posthog-js'
+import {
+  FilesetResolver,
+  FaceLandmarker,
+  FaceLandmarkerResult,
+} from '@mediapipe/tasks-vision'
+import {
+  CLEAR_TIMEOUT,
+  SET_TIMEOUT,
+  TIMEOUT_TICK,
+  timerWorkerScript,
+} from './TimerWorker'
+import {
+  BackgroundProcessorInterface,
+  BackgroundOptions,
+  ProcessorType,
+} from '.'
+
+const PROCESSING_WIDTH = 256 * 3
+const PROCESSING_HEIGHT = 144 * 3
+
+const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local'
+
+export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
+  options: BackgroundOptions
+  name: string
+  processedTrack?: MediaStreamTrack | undefined
+
+  source?: MediaStreamTrack
+  sourceSettings?: MediaTrackSettings
+  videoElement?: HTMLVideoElement
+  videoElementLoaded?: boolean
+
+  // Canvas containing the video processing result
+  outputCanvas?: HTMLCanvasElement
+  outputCanvasCtx?: CanvasRenderingContext2D
+
+  faceLandmarker?: FaceLandmarker
+  faceLandmarkerResult?: FaceLandmarkerResult
+
+  // The resized image of the video source
+  sourceImageData?: ImageData
+
+  timerWorker?: Worker
+
+  type: ProcessorType
+
+  constructor(opts: BackgroundOptions) {
+    this.name = 'face_landmarks'
+    this.options = opts
+    this.type = ProcessorType.FACE_LANDMARKS
+  }
+
+  static get isSupported() {
+    return true // Face landmarks should work in all modern browsers
+  }
+
+  async init(opts: ProcessorOptions<Track.Kind>) {
+    if (!opts.element) {
+      throw new Error('Element is required for processing')
+    }
+
+    this.source = opts.track as MediaStreamTrack
+    this.sourceSettings = this.source!.getSettings()
+    this.videoElement = opts.element as HTMLVideoElement
+
+    this._createMainCanvas()
+
+    const stream = this.outputCanvas!.captureStream()
+    const tracks = stream.getVideoTracks()
+    if (tracks.length == 0) {
+      throw new Error('No tracks found for processing')
+    }
+    this.processedTrack = tracks[0]
+
+    await this.initFaceLandmarker()
+    this._initWorker()
+
+    posthog.capture('face-landmarks-init')
+  }
+
+  _initWorker() {
+    this.timerWorker = new Worker(timerWorkerScript, {
+      name: 'FaceLandmarks',
+    })
+    this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
+    if (this.videoElementLoaded) {
+      this.timerWorker!.postMessage({
+        id: SET_TIMEOUT,
+        timeMs: 1000 / 30,
+      })
+    } else {
+      this.videoElement!.onloadeddata = () => {
+        this.videoElementLoaded = true
+        this.timerWorker!.postMessage({
+          id: SET_TIMEOUT,
+          timeMs: 1000 / 30,
+        })
+      }
+    }
+  }
+
+  onTimerMessage(response: { data: { id: number } }) {
+    if (response.data.id === TIMEOUT_TICK) {
+      this.process()
+    }
+  }
+
+  async initFaceLandmarker() {
+    const vision = await FilesetResolver.forVisionTasks(
+      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
+    )
+    this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
+      baseOptions: {
+        modelAssetPath:
+          'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',
+        delegate: 'GPU',
+      },
+      runningMode: 'VIDEO',
+      outputFaceBlendshapes: true,
+      outputFacialTransformationMatrixes: true,
+    })
+  }
+
+  async sizeSource() {
+    this.outputCanvasCtx?.drawImage(
+      this.videoElement!,
+      0,
+      0,
+      this.videoElement!.videoWidth,
+      this.videoElement!.videoHeight,
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT
+    )
+
+    this.sourceImageData = this.outputCanvasCtx?.getImageData(
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT
+    )
+  }
+
+  async detectFaces() {
+    const startTimeMs = performance.now()
+    this.faceLandmarkerResult = this.faceLandmarker!.detectForVideo(
+      this.sourceImageData!,
+      startTimeMs
+    )
+  }
+
+  async drawFaceLandmarks() {
+    // Draw the original video frame at the canvas size
+    this.outputCanvasCtx!.drawImage(
+      this.videoElement!,
+      0,
+      0,
+      this.videoElement!.videoWidth,
+      this.videoElement!.videoHeight,
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT
+    )
+
+    if (!this.faceLandmarkerResult?.faceLandmarks) {
+      return
+    }
+
+    // Draw face landmarks
+    this.outputCanvasCtx!.strokeStyle = '#00FF00'
+    this.outputCanvasCtx!.lineWidth = 2
+
+    for (const face of this.faceLandmarkerResult.faceLandmarks) {
+      for (const landmark of face) {
+        // Use the same dimensions as the canvas/video display size
+        const x = landmark.x * PROCESSING_WIDTH
+        const y = landmark.y * PROCESSING_HEIGHT
+
+        this.outputCanvasCtx!.beginPath()
+        this.outputCanvasCtx!.arc(x, y, 2, 0, 2 * Math.PI)
+        this.outputCanvasCtx!.stroke()
+      }
+    }
+  }
+
+  async process() {
+    await this.sizeSource()
+    await this.detectFaces()
+    await this.drawFaceLandmarks()
+
+    this.timerWorker!.postMessage({
+      id: SET_TIMEOUT,
+      timeMs: 1000 / 30,
+    })
+  }
+
+  _createMainCanvas() {
+    this.outputCanvas = document.querySelector(
+      `#${FACE_LANDMARKS_CANVAS_ID}`
+    ) as HTMLCanvasElement
+    if (!this.outputCanvas) {
+      this.outputCanvas = this._createCanvas(
+        FACE_LANDMARKS_CANVAS_ID,
+        PROCESSING_WIDTH,
+        PROCESSING_HEIGHT
+      )
+    }
+    this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
+  }
+
+  _createCanvas(id: string, width: number, height: number) {
+    const element = document.createElement('canvas')
+    element.setAttribute('id', id)
+    element.setAttribute('width', '' + width)
+    element.setAttribute('height', '' + height)
+    return element
+  }
+
+  update(opts: BackgroundOptions): void {
+    this.options = opts
+  }
+
+  async restart(opts: ProcessorOptions<Track.Kind>) {
+    await this.destroy()
+    return this.init(opts)
+  }
+
+  async destroy() {
+    this.timerWorker?.postMessage({
+      id: CLEAR_TIMEOUT,
+    })
+
+    this.timerWorker?.terminate()
+    this.faceLandmarker?.close()
+  }
+
+  clone() {
+    return new FaceLandmarksProcessor(this.options)
+  }
+
+  serialize() {
+    return {
+      type: this.type,
+      options: this.options,
+    }
+  }
+} 

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -3,10 +3,12 @@ import { Track, TrackProcessor } from 'livekit-client'
 import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
 import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
 import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'
+import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
 
 export type BackgroundOptions = {
   blurRadius?: number
   imagePath?: string
+  showFaceLandmarks?: boolean
 }
 
 export interface ProcessorSerialized {
@@ -25,11 +27,12 @@ export interface BackgroundProcessorInterface
 export enum ProcessorType {
   BLUR = 'blur',
   VIRTUAL = 'virtual',
+  FACE_LANDMARKS = 'faceLandmarks'
 }
 
 export class BackgroundProcessorFactory {
   static isSupported() {
-    return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported
+    return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported || FaceLandmarksProcessor.isSupported
   }
 
   static getProcessor(
@@ -50,6 +53,10 @@ export class BackgroundProcessorFactory {
       if (BackgroundCustomProcessor.isSupported) {
         return new BackgroundCustomProcessor(opts)
       }
+    } else if (type === ProcessorType.FACE_LANDMARKS) {
+      if (FaceLandmarksProcessor.isSupported) {
+        return new FaceLandmarksProcessor(opts)
+      }
     }
     return undefined
   }

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -15,7 +15,7 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
 import { useTrackToggle } from '@livekit/components-react'
 import { Loader } from '@/primitives/Loader'
 import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import { RiProhibited2Line } from '@remixicon/react'
+import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'
 
 enum BlurRadius {
   NONE = 0,
@@ -302,6 +302,49 @@ export const EffectsConfiguration = ({
                   </ToggleButton>
                 </div>
               </div>
+              <div
+                className={css({
+                  marginTop: '1.5rem',
+                })}
+              >
+                <H
+                  lvl={3}
+                  style={{
+                    marginBottom: '1rem',
+                  }}
+                  variant="bodyXsBold"
+                >
+                  {t('faceLandmarks.title')}
+                </H>
+                <div
+                  className={css({
+                    display: 'flex',
+                    gap: '1.25rem',
+                  })}
+                >
+                  <ToggleButton
+                    variant="bigSquare"
+                    aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                      blurRadius: 0,
+                    })}
+                    tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                      blurRadius: 0,
+                    })}
+                    isDisabled={processorPendingReveal}
+                    onChange={async () =>
+                      await toggleEffect(ProcessorType.FACE_LANDMARKS, {
+                        blurRadius: 0,
+                      })
+                    }
+                    isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {
+                      blurRadius: 0,
+                    })}
+                    data-attr="toggle-face-landmarks"
+                  >
+                    <RiUserVoiceLine />
+                  </ToggleButton>
+                </div>
+              </div>
               <div
                 className={css({
                   marginTop: '1.5rem',

src/frontend/src/locales/de/rooms.json

@@ -135,22 +135,27 @@
     }
   },
   "effects": {
-    "activateCamera": "",
+    "activateCamera": "Ihre Kamera ist deaktiviert. Wählen Sie eine Option, um sie zu aktivieren.",
-    "notAvailable": "",
+    "notAvailable": "Videoeffekte werden in Kürze in Ihrem Browser verfügbar sein. Wir arbeiten daran! In der Zwischenzeit können Sie Google Chrome für beste Leistung oder Firefox verwenden :(",
-    "heading": "",
+    "heading": "Unschärfe",
     "blur": {
-      "title": "",
+      "title": "Hintergrundunschärfe",
-      "light": "",
+      "light": "Leichte Unschärfe",
-      "normal": "",
+      "normal": "Unschärfe",
-      "apply": "",
+      "apply": "Unschärfe aktivieren",
-      "clear": ""
+      "clear": "Unschärfe deaktivieren"
     },
     "virtual": {
-      "title": "",
+      "title": "Virtueller Hintergrund",
-      "apply": "",
+      "apply": "Virtuellen Hintergrund aktivieren",
-      "clear": ""
+      "clear": "Virtuellen Hintergrund deaktivieren"
     },
-    "experimental": ""
+    "faceLandmarks": {
+      "title": "Gesichtsmerkmale",
+      "apply": "Gesichtsmerkmale aktivieren",
+      "clear": "Gesichtsmerkmale deaktivieren"
+    },
+    "experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."
   },
   "sidePanel": {
     "heading": {

src/frontend/src/locales/en/rooms.json

@@ -149,6 +149,11 @@
       "apply": "Enable virtual background",
       "clear": "Disable virtual background"
     },
+    "faceLandmarks": {
+      "title": "Face landmarks",
+      "apply": "Enable face landmarks",
+      "clear": "Disable face landmarks"
+    },
     "experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."
   },
   "sidePanel": {

src/frontend/src/locales/fr/rooms.json

@@ -134,22 +134,27 @@
     }
   },
   "effects": {
-    "activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.",
+    "activateCamera": "Votre caméra est désactivée. Choisissez une option pour l'activer.",
-    "notAvailable": "Les effets vidéo seront bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour une meilleure performance ou Firefox :(",
+    "notAvailable": "Les effets vidéo seront bientôt disponibles sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour de meilleures performances ou Firefox :(",
     "heading": "Flou",
     "blur": {
       "title": "Flou d'arrière-plan",
-      "light": "Léger flou",
+      "light": "Flou léger",
       "normal": "Flou",
-      "apply": "Appliquer le flou",
+      "apply": "Activer le flou",
       "clear": "Désactiver le flou"
     },
     "virtual": {
       "title": "Arrière-plan virtuel",
-      "apply": "Appliquer l'arrière plan virtuel",
+      "apply": "Activer l'arrière-plan virtuel",
-      "clear": "Désactiver l'arrière plan virtuel"
+      "clear": "Désactiver l'arrière-plan virtuel"
     },
-    "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet sur tous les navigateurs et une meilleur qualité."
+    "faceLandmarks": {
+      "title": "Points du visage",
+      "apply": "Activer les points du visage",
+      "clear": "Désactiver les points du visage"
+    },
+    "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."
   },
   "sidePanel": {
     "heading": {

src/frontend/src/locales/nl/rooms.json

@@ -134,22 +134,31 @@
     }
   },
   "effects": {
-    "activateCamera": "Uw camera is uitgeschakeld. Kies een optie om hem in te schakelen.",
+    "activateCamera": "Uw camera is uitgeschakeld. Kies een optie om deze in te schakelen.",
-    "notAvailable": "Video-effecten zijn binnenkort beschikbaar in uw browser. We werken hier aan! Ondertussen kunt u Chrome gebruiken voor de beste prestaties, of Firefox :(",
+    "notAvailable": "Video-effecten zijn binnenkort beschikbaar in uw browser. We werken eraan! In de tussentijd kunt u Google Chrome gebruiken voor de beste prestaties of Firefox :(",
     "heading": "Vervaging",
     "blur": {
-      "title": "Achtergrond vervagen",
+      "title": "Achtergrondvervaging",
-      "light": "Licht vervagen",
+      "light": "Lichte vervaging",
-      "normal": "Vervagen",
+      "normal": "Vervaging",
-      "apply": "Schakel vervaging in",
+      "apply": "Vervaging inschakelen",
-      "clear": "Schakel vervaging uit"
+      "clear": "Vervaging uitschakelen"
     },
     "virtual": {
       "title": "Virtuele achtergrond",
-      "apply": "Schakel virtuele achtergrond in",
+      "apply": "Virtuele achtergrond inschakelen",
-      "clear": "Schakel virtuele achtergrond uit"
+      "clear": "Virtuele achtergrond uitschakelen"
     },
-    "experimental": "Experimentele functionaliteit. Een v2 komt er aan met volledige browser-ondersteuning en verbeterde kwaliteit."
+    "faceLandmarks": {
+      "title": "Gezichtskenmerken",
+      "apply": "Gezichtskenmerken inschakelen",
+      "clear": "Gezichtskenmerken uitschakelen",
+      "tooltip": {
+        "apply": "Gezichtskenmerken inschakelen",
+        "clear": "Gezichtskenmerken uitschakelen"
+      }
+    },
+    "experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."
   },
   "sidePanel": {
     "heading": {