✨(frontend) add mustache effects

Added functionality to display mustache effects
on detected faces. Enhanced the EffectsConfiguration
component to toggle these effects and updated
localization files for this effects.

src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts

@@ -46,20 +46,25 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
 
   type: ProcessorType
 
-  // Glasses image element
+  // Effect images
   glassesImage?: HTMLImageElement
+  mustacheImage?: HTMLImageElement
 
   constructor(opts: BackgroundOptions) {
     this.name = 'face_landmarks'
     this.options = opts
     this.type = ProcessorType.FACE_LANDMARKS
-    this._initGlassesImage()
+    this._initEffectImages()
   }
 
-  private _initGlassesImage() {
+  private _initEffectImages() {
     this.glassesImage = new Image()
-    this.glassesImage.src = '/assets/glasses.png' // You'll need to add this image to your public assets
+    this.glassesImage.src = '/assets/glasses.png'
     this.glassesImage.crossOrigin = 'anonymous'
+
+    this.mustacheImage = new Image()
+    this.mustacheImage.src = '/assets/mustache.png'
+    this.mustacheImage.crossOrigin = 'anonymous'
   }
 
   static get isSupported() {
@@ -162,6 +167,53 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
     )
   }
 
+  private drawEffect(
+    leftPoint: { x: number; y: number },
+    rightPoint: { x: number; y: number },
+    image: HTMLImageElement,
+    widthScale: number,
+    heightScale: number
+  ) {
+    // Calculate distance between points
+    const distance = Math.sqrt(
+      Math.pow(rightPoint.x - leftPoint.x, 2) + 
+      Math.pow(rightPoint.y - leftPoint.y, 2)
+    )
+    
+    // Scale image based on distance
+    const width = distance * PROCESSING_WIDTH * widthScale
+    const height = width * heightScale
+    
+    // Calculate center position between points
+    const centerX = (leftPoint.x + rightPoint.x) / 2
+    const centerY = (leftPoint.y + rightPoint.y) / 2
+    
+    // Draw image
+    this.outputCanvasCtx!.save()
+    this.outputCanvasCtx!.translate(
+      centerX * PROCESSING_WIDTH,
+      centerY * PROCESSING_HEIGHT
+    )
+    
+    // Calculate rotation angle based on point positions
+    const angle = Math.atan2(
+      rightPoint.y - leftPoint.y,
+      rightPoint.x - leftPoint.x
+    )
+    this.outputCanvasCtx!.rotate(angle)
+    
+    // Draw image centered at the midpoint between points
+    this.outputCanvasCtx!.drawImage(
+      image,
+      -width / 2,
+      -height / 2,
+      width,
+      height
+    )
+    
+    this.outputCanvasCtx!.restore()
+  }
+
   async drawFaceLandmarks() {
     // Draw the original video frame at the canvas size
     this.outputCanvasCtx!.drawImage(
@@ -185,49 +237,20 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
     this.outputCanvasCtx!.lineWidth = 2
 
     for (const face of this.faceLandmarkerResult.faceLandmarks) {
-      // Find eye landmarks (indices 33 and 263 are the left and right eye centers)
+      // Find eye landmarks (indices 33 and 263 are the left and right eye corners)
-      const leftEye = face[33]
+      const leftEye = face[468]
-      const rightEye = face[263]
+      const rightEye = face[473]
       
-      if (leftEye && rightEye) {
+      // Find mouth landmarks for mustache (indices 0 and 17 are the left and right corners of the mouth)
-        // Calculate glasses position and size
+      const leftMoustache = face[92]
-        const eyeDistance = Math.sqrt(
+      const rightMoustache = face[322]
-          Math.pow(rightEye.x - leftEye.x, 2) + 
-          Math.pow(rightEye.y - leftEye.y, 2)
-        )
       
-        // Scale glasses based on eye distance
+      if (leftEye && rightEye && this.options.showGlasses) {
-        const glassesWidth = eyeDistance * PROCESSING_WIDTH * 2.5 // Adjust multiplier as needed
+        this.drawEffect(leftEye, rightEye, this.glassesImage!, 2.5, 0.7)
-        const glassesHeight = glassesWidth * 0.7 // Adjust aspect ratio as needed
+      }
 
-        // Calculate center position between eyes
+      if (leftMoustache && rightMoustache && this.options.showMustache) {
-        const centerX = (leftEye.x + rightEye.x) / 2
+        this.drawEffect(leftMoustache, rightMoustache, this.mustacheImage!, 1.5, 0.5)
-        const centerY = (leftEye.y + rightEye.y) / 2
-        
-        // Draw glasses
-        this.outputCanvasCtx!.save()
-        this.outputCanvasCtx!.translate(
-          centerX * PROCESSING_WIDTH,
-          centerY * PROCESSING_HEIGHT
-        )
-        
-        // Calculate rotation angle based on eye positions
-        const angle = Math.atan2(
-          rightEye.y - leftEye.y,
-          rightEye.x - leftEye.x
-        )
-        this.outputCanvasCtx!.rotate(angle)
-        
-        // Draw glasses centered at the midpoint between eyes
-        this.outputCanvasCtx!.drawImage(
-          this.glassesImage!,
-          -glassesWidth / 2,
-          -glassesHeight / 2,
-          glassesWidth,
-          glassesHeight
-        )
-        
-        this.outputCanvasCtx!.restore()
       }
     }
   }

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -8,7 +8,8 @@ import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
 export type BackgroundOptions = {
   blurRadius?: number
   imagePath?: string
-  showFaceLandmarks?: boolean
+  showGlasses?: boolean
+  showMustache?: boolean
 }
 
 export interface ProcessorSerialized {

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -5,17 +5,17 @@ import {
   BackgroundProcessorFactory,
   BackgroundProcessorInterface,
   ProcessorType,
+  BackgroundOptions
 } from '../blur'
 import { css } from '@/styled-system/css'
 import { Text, P, ToggleButton, H } from '@/primitives'
 import { styled } from '@/styled-system/jsx'
-import { BackgroundOptions } from '@livekit/track-processors'
 import { BlurOn } from '@/components/icons/BlurOn'
 import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
 import { useTrackToggle } from '@livekit/components-react'
 import { Loader } from '@/primitives/Loader'
 import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'
+import { RiProhibited2Line, RiGlassesLine, RiEmotionLine } from '@remixicon/react'
 import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'
 
 enum BlurRadius {
@@ -141,9 +141,36 @@ export const EffectsConfiguration = ({
   }
 
   const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {
+    if (type === ProcessorType.FACE_LANDMARKS) {
+      const effect = options.showGlasses ? 'glasses' : 'mustache'
+      return t(`faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`)
+    }
     return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)
   }
 
+  const getFaceLandmarksOptions = () => {
+    const processor = getProcessor()
+    if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {
+      return processor.serialize().options as { showGlasses?: boolean; showMustache?: boolean }
+    }
+    return { showGlasses: false, showMustache: false }
+  }
+
+  const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'mustache') => {
+    const currentOptions = getFaceLandmarksOptions()
+    const newOptions = {
+      ...currentOptions,
+      [effect === 'glasses' ? 'showGlasses' : 'showMustache']: !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showMustache']
+    }
+
+    if (!newOptions.showGlasses && !newOptions.showMustache) {
+      // If both effects are off stop the processor
+      await clearEffect()
+    } else {
+      await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)
+    }
+  }
+
   return (
     <div
       className={css(
@@ -328,23 +355,36 @@ export const EffectsConfiguration = ({
                     <ToggleButton
                       variant="bigSquare"
                       aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
+                        showGlasses: true,
+                        showMustache: false,
                       })}
                       tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
+                        showGlasses: true,
+                        showMustache: false,
                       })}
                       isDisabled={processorPendingReveal}
-                      onChange={async () =>
+                      onChange={async () => await toggleFaceLandmarkEffect('glasses')}
-                        await toggleEffect(ProcessorType.FACE_LANDMARKS, {
+                      isSelected={getFaceLandmarksOptions().showGlasses}
-                          blurRadius: 0,
+                      data-attr="toggle-glasses"
-                        })
-                      }
-                      isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
-                      })}
-                      data-attr="toggle-face-landmarks"
                     >
-                      <RiUserVoiceLine />
+                      <RiGlassesLine />
+                    </ToggleButton>
+                    <ToggleButton
+                      variant="bigSquare"
+                      aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                        showGlasses: false,
+                        showMustache: true,
+                      })}
+                      tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                        showGlasses: false,
+                        showMustache: true,
+                      })}
+                      isDisabled={processorPendingReveal}
+                      onChange={async () => await toggleFaceLandmarkEffect('mustache')}
+                      isSelected={getFaceLandmarksOptions().showMustache}
+                      data-attr="toggle-mustache"
+                    >
+                      <RiEmotionLine />
                     </ToggleButton>
                   </div>
                 </div>

src/frontend/src/locales/de/rooms.json

@@ -151,9 +151,15 @@
       "clear": "Virtuellen Hintergrund deaktivieren"
     },
     "faceLandmarks": {
-      "title": "Gesichtsmerkmale",
+      "title": "Visuelle Effekte",
-      "apply": "Gesichtsmerkmale aktivieren",
+      "glasses": {
-      "clear": "Gesichtsmerkmale deaktivieren"
+        "apply": "Brille hinzufügen",
+        "clear": "Brille entfernen"
+      },
+      "mustache": {
+        "apply": "Schnurrbart hinzufügen",
+        "clear": "Schnurrbart entfernen"
+      }
     },
     "experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."
   },

src/frontend/src/locales/en/rooms.json

@@ -150,9 +150,15 @@
       "clear": "Disable virtual background"
     },
     "faceLandmarks": {
-      "title": "Face landmarks",
+      "title": "Visual Effects",
-      "apply": "Enable face landmarks",
+      "glasses": {
-      "clear": "Disable face landmarks"
+        "apply": "Add Glasses",
+        "clear": "Remove Glasses"
+      },
+      "mustache": {
+        "apply": "Add Mustache",
+        "clear": "Remove Mustache"
+      }
     },
     "experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."
   },

src/frontend/src/locales/fr/rooms.json

@@ -150,9 +150,15 @@
       "clear": "Désactiver l'arrière-plan virtuel"
     },
     "faceLandmarks": {
-      "title": "Points du visage",
+      "title": "Effets visuels",
-      "apply": "Activer les points du visage",
+      "glasses": {
-      "clear": "Désactiver les points du visage"
+        "apply": "Ajouter des lunettes",
+        "clear": "Retirer les lunettes"
+      },
+      "mustache": {
+        "apply": "Ajouter une moustache",
+        "clear": "Retirer la moustache"
+      }
     },
     "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."
   },

src/frontend/src/locales/nl/rooms.json

@@ -150,12 +150,14 @@
       "clear": "Virtuele achtergrond uitschakelen"
     },
     "faceLandmarks": {
-      "title": "Gezichtskenmerken",
+      "title": "Visuele effecten",
-      "apply": "Gezichtskenmerken inschakelen",
+      "glasses": {
-      "clear": "Gezichtskenmerken uitschakelen",
+        "apply": "Bril toevoegen",
-      "tooltip": {
+        "clear": "Bril verwijderen"
-        "apply": "Gezichtskenmerken inschakelen",
+      },
-        "clear": "Gezichtskenmerken uitschakelen"
+      "mustache": {
+        "apply": "Snor toevoegen",
+        "clear": "Snor verwijderen"
       }
     },
     "experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."