✨(frontend) add mustache effects

Added functionality to display mustache effects on detected faces. Enhanced the EffectsConfiguration component to toggle these effects and updated localization files for this effects.
2025-03-31 00:23:57 +02:00
parent 0d6881382b
commit 1d5aebcfdc
8 changed files with 160 additions and 76 deletions
--- a/src/frontend/public/assets/mustache.png
+++ b/src/frontend/public/assets/mustache.png
--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
@@ -46,20 +46,25 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {

  type: ProcessorType

-  // Glasses image element
+  // Effect images
  glassesImage?: HTMLImageElement
+  mustacheImage?: HTMLImageElement

  constructor(opts: BackgroundOptions) {
    this.name = 'face_landmarks'
    this.options = opts
    this.type = ProcessorType.FACE_LANDMARKS
-    this._initGlassesImage()
+    this._initEffectImages()
  }

-  private _initGlassesImage() {
+  private _initEffectImages() {
    this.glassesImage = new Image()
-    this.glassesImage.src = '/assets/glasses.png' // You'll need to add this image to your public assets
+    this.glassesImage.src = '/assets/glasses.png'
    this.glassesImage.crossOrigin = 'anonymous'
+
+    this.mustacheImage = new Image()
+    this.mustacheImage.src = '/assets/mustache.png'
+    this.mustacheImage.crossOrigin = 'anonymous'
  }

  static get isSupported() {
@@ -162,6 +167,53 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
    )
  }

+  private drawEffect(
+    leftPoint: { x: number; y: number },
+    rightPoint: { x: number; y: number },
+    image: HTMLImageElement,
+    widthScale: number,
+    heightScale: number
+  ) {
+    // Calculate distance between points
+    const distance = Math.sqrt(
+      Math.pow(rightPoint.x - leftPoint.x, 2) + 
+      Math.pow(rightPoint.y - leftPoint.y, 2)
+    )
+    
+    // Scale image based on distance
+    const width = distance * PROCESSING_WIDTH * widthScale
+    const height = width * heightScale
+    
+    // Calculate center position between points
+    const centerX = (leftPoint.x + rightPoint.x) / 2
+    const centerY = (leftPoint.y + rightPoint.y) / 2
+    
+    // Draw image
+    this.outputCanvasCtx!.save()
+    this.outputCanvasCtx!.translate(
+      centerX * PROCESSING_WIDTH,
+      centerY * PROCESSING_HEIGHT
+    )
+    
+    // Calculate rotation angle based on point positions
+    const angle = Math.atan2(
+      rightPoint.y - leftPoint.y,
+      rightPoint.x - leftPoint.x
+    )
+    this.outputCanvasCtx!.rotate(angle)
+    
+    // Draw image centered at the midpoint between points
+    this.outputCanvasCtx!.drawImage(
+      image,
+      -width / 2,
+      -height / 2,
+      width,
+      height
+    )
+    
+    this.outputCanvasCtx!.restore()
+  }
+
  async drawFaceLandmarks() {
    // Draw the original video frame at the canvas size
    this.outputCanvasCtx!.drawImage(
@@ -185,49 +237,20 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
    this.outputCanvasCtx!.lineWidth = 2

    for (const face of this.faceLandmarkerResult.faceLandmarks) {
-      // Find eye landmarks (indices 33 and 263 are the left and right eye centers)
-      const leftEye = face[33]
-      const rightEye = face[263]
+      // Find eye landmarks (indices 33 and 263 are the left and right eye corners)
+      const leftEye = face[468]
+      const rightEye = face[473]
      
-      if (leftEye && rightEye) {
-        // Calculate glasses position and size
-        const eyeDistance = Math.sqrt(
-          Math.pow(rightEye.x - leftEye.x, 2) + 
-          Math.pow(rightEye.y - leftEye.y, 2)
-        )
-        
-        // Scale glasses based on eye distance
-        const glassesWidth = eyeDistance * PROCESSING_WIDTH * 2.5 // Adjust multiplier as needed
-        const glassesHeight = glassesWidth * 0.7 // Adjust aspect ratio as needed
-        
-        // Calculate center position between eyes
-        const centerX = (leftEye.x + rightEye.x) / 2
-        const centerY = (leftEye.y + rightEye.y) / 2
-        
-        // Draw glasses
-        this.outputCanvasCtx!.save()
-        this.outputCanvasCtx!.translate(
-          centerX * PROCESSING_WIDTH,
-          centerY * PROCESSING_HEIGHT
-        )
-        
-        // Calculate rotation angle based on eye positions
-        const angle = Math.atan2(
-          rightEye.y - leftEye.y,
-          rightEye.x - leftEye.x
-        )
-        this.outputCanvasCtx!.rotate(angle)
-        
-        // Draw glasses centered at the midpoint between eyes
-        this.outputCanvasCtx!.drawImage(
-          this.glassesImage!,
-          -glassesWidth / 2,
-          -glassesHeight / 2,
-          glassesWidth,
-          glassesHeight
-        )
-        
-        this.outputCanvasCtx!.restore()
+      // Find mouth landmarks for mustache (indices 0 and 17 are the left and right corners of the mouth)
+      const leftMoustache = face[92]
+      const rightMoustache = face[322]
+      
+      if (leftEye && rightEye && this.options.showGlasses) {
+        this.drawEffect(leftEye, rightEye, this.glassesImage!, 2.5, 0.7)
+      }
+
+      if (leftMoustache && rightMoustache && this.options.showMustache) {
+        this.drawEffect(leftMoustache, rightMoustache, this.mustacheImage!, 1.5, 0.5)
      }
    }
  }
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -8,7 +8,8 @@ import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
 export type BackgroundOptions = {
  blurRadius?: number
  imagePath?: string
-  showFaceLandmarks?: boolean
+  showGlasses?: boolean
+  showMustache?: boolean
 }

 export interface ProcessorSerialized {
--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
@@ -5,17 +5,17 @@ import {
  BackgroundProcessorFactory,
  BackgroundProcessorInterface,
  ProcessorType,
+  BackgroundOptions
 } from '../blur'
 import { css } from '@/styled-system/css'
 import { Text, P, ToggleButton, H } from '@/primitives'
 import { styled } from '@/styled-system/jsx'
-import { BackgroundOptions } from '@livekit/track-processors'
 import { BlurOn } from '@/components/icons/BlurOn'
 import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
 import { useTrackToggle } from '@livekit/components-react'
 import { Loader } from '@/primitives/Loader'
 import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'
+import { RiProhibited2Line, RiGlassesLine, RiEmotionLine } from '@remixicon/react'
 import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'

 enum BlurRadius {
@@ -141,9 +141,36 @@ export const EffectsConfiguration = ({
  }

  const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {
+    if (type === ProcessorType.FACE_LANDMARKS) {
+      const effect = options.showGlasses ? 'glasses' : 'mustache'
+      return t(`faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`)
+    }
    return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)
  }

+  const getFaceLandmarksOptions = () => {
+    const processor = getProcessor()
+    if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {
+      return processor.serialize().options as { showGlasses?: boolean; showMustache?: boolean }
+    }
+    return { showGlasses: false, showMustache: false }
+  }
+
+  const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'mustache') => {
+    const currentOptions = getFaceLandmarksOptions()
+    const newOptions = {
+      ...currentOptions,
+      [effect === 'glasses' ? 'showGlasses' : 'showMustache']: !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showMustache']
+    }
+
+    if (!newOptions.showGlasses && !newOptions.showMustache) {
+      // If both effects are off stop the processor
+      await clearEffect()
+    } else {
+      await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)
+    }
+  }
+
  return (
    <div
      className={css(
@@ -328,23 +355,36 @@ export const EffectsConfiguration = ({
                    <ToggleButton
                      variant="bigSquare"
                      aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
+                        showGlasses: true,
+                        showMustache: false,
                      })}
                      tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
+                        showGlasses: true,
+                        showMustache: false,
                      })}
                      isDisabled={processorPendingReveal}
-                      onChange={async () =>
-                        await toggleEffect(ProcessorType.FACE_LANDMARKS, {
-                          blurRadius: 0,
-                        })
-                      }
-                      isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {
-                        blurRadius: 0,
-                      })}
-                      data-attr="toggle-face-landmarks"
+                      onChange={async () => await toggleFaceLandmarkEffect('glasses')}
+                      isSelected={getFaceLandmarksOptions().showGlasses}
+                      data-attr="toggle-glasses"
                    >
-                      <RiUserVoiceLine />
+                      <RiGlassesLine />
+                    </ToggleButton>
+                    <ToggleButton
+                      variant="bigSquare"
+                      aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                        showGlasses: false,
+                        showMustache: true,
+                      })}
+                      tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+                        showGlasses: false,
+                        showMustache: true,
+                      })}
+                      isDisabled={processorPendingReveal}
+                      onChange={async () => await toggleFaceLandmarkEffect('mustache')}
+                      isSelected={getFaceLandmarksOptions().showMustache}
+                      data-attr="toggle-mustache"
+                    >
+                      <RiEmotionLine />
                    </ToggleButton>
                  </div>
                </div>
--- a/src/frontend/src/locales/de/rooms.json
+++ b/src/frontend/src/locales/de/rooms.json
@@ -151,9 +151,15 @@
      "clear": "Virtuellen Hintergrund deaktivieren"
    },
    "faceLandmarks": {
-      "title": "Gesichtsmerkmale",
-      "apply": "Gesichtsmerkmale aktivieren",
-      "clear": "Gesichtsmerkmale deaktivieren"
+      "title": "Visuelle Effekte",
+      "glasses": {
+        "apply": "Brille hinzufügen",
+        "clear": "Brille entfernen"
+      },
+      "mustache": {
+        "apply": "Schnurrbart hinzufügen",
+        "clear": "Schnurrbart entfernen"
+      }
    },
    "experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."
  },
--- a/src/frontend/src/locales/en/rooms.json
+++ b/src/frontend/src/locales/en/rooms.json
@@ -150,9 +150,15 @@
      "clear": "Disable virtual background"
    },
    "faceLandmarks": {
-      "title": "Face landmarks",
-      "apply": "Enable face landmarks",
-      "clear": "Disable face landmarks"
+      "title": "Visual Effects",
+      "glasses": {
+        "apply": "Add Glasses",
+        "clear": "Remove Glasses"
+      },
+      "mustache": {
+        "apply": "Add Mustache",
+        "clear": "Remove Mustache"
+      }
    },
    "experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."
  },
--- a/src/frontend/src/locales/fr/rooms.json
+++ b/src/frontend/src/locales/fr/rooms.json
@@ -150,9 +150,15 @@
      "clear": "Désactiver l'arrière-plan virtuel"
    },
    "faceLandmarks": {
-      "title": "Points du visage",
-      "apply": "Activer les points du visage",
-      "clear": "Désactiver les points du visage"
+      "title": "Effets visuels",
+      "glasses": {
+        "apply": "Ajouter des lunettes",
+        "clear": "Retirer les lunettes"
+      },
+      "mustache": {
+        "apply": "Ajouter une moustache",
+        "clear": "Retirer la moustache"
+      }
    },
    "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."
  },
--- a/src/frontend/src/locales/nl/rooms.json
+++ b/src/frontend/src/locales/nl/rooms.json
@@ -150,12 +150,14 @@
      "clear": "Virtuele achtergrond uitschakelen"
    },
    "faceLandmarks": {
-      "title": "Gezichtskenmerken",
-      "apply": "Gezichtskenmerken inschakelen",
-      "clear": "Gezichtskenmerken uitschakelen",
-      "tooltip": {
-        "apply": "Gezichtskenmerken inschakelen",
-        "clear": "Gezichtskenmerken uitschakelen"
+      "title": "Visuele effecten",
+      "glasses": {
+        "apply": "Bril toevoegen",
+        "clear": "Bril verwijderen"
+      },
+      "mustache": {
+        "apply": "Snor toevoegen",
+        "clear": "Snor verwijderen"
      }
    },
    "experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."