diff --git a/src/frontend/public/assets/mustache.png b/src/frontend/public/assets/mustache.png new file mode 100644 index 00000000..09956bd4 Binary files /dev/null and b/src/frontend/public/assets/mustache.png differ diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts index 6661c487..462ce624 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts @@ -46,20 +46,25 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface { type: ProcessorType - // Glasses image element + // Effect images glassesImage?: HTMLImageElement + mustacheImage?: HTMLImageElement constructor(opts: BackgroundOptions) { this.name = 'face_landmarks' this.options = opts this.type = ProcessorType.FACE_LANDMARKS - this._initGlassesImage() + this._initEffectImages() } - private _initGlassesImage() { + private _initEffectImages() { this.glassesImage = new Image() - this.glassesImage.src = '/assets/glasses.png' // You'll need to add this image to your public assets + this.glassesImage.src = '/assets/glasses.png' this.glassesImage.crossOrigin = 'anonymous' + + this.mustacheImage = new Image() + this.mustacheImage.src = '/assets/mustache.png' + this.mustacheImage.crossOrigin = 'anonymous' } static get isSupported() { @@ -162,6 +167,53 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface { ) } + private drawEffect( + leftPoint: { x: number; y: number }, + rightPoint: { x: number; y: number }, + image: HTMLImageElement, + widthScale: number, + heightScale: number + ) { + // Calculate distance between points + const distance = Math.sqrt( + Math.pow(rightPoint.x - leftPoint.x, 2) + + Math.pow(rightPoint.y - leftPoint.y, 2) + ) + + // Scale image based on distance + const width = distance * PROCESSING_WIDTH * widthScale + const height = width * heightScale + + // Calculate center position between points + const centerX = (leftPoint.x + rightPoint.x) / 2 + const centerY = (leftPoint.y + rightPoint.y) / 2 + + // Draw image + this.outputCanvasCtx!.save() + this.outputCanvasCtx!.translate( + centerX * PROCESSING_WIDTH, + centerY * PROCESSING_HEIGHT + ) + + // Calculate rotation angle based on point positions + const angle = Math.atan2( + rightPoint.y - leftPoint.y, + rightPoint.x - leftPoint.x + ) + this.outputCanvasCtx!.rotate(angle) + + // Draw image centered at the midpoint between points + this.outputCanvasCtx!.drawImage( + image, + -width / 2, + -height / 2, + width, + height + ) + + this.outputCanvasCtx!.restore() + } + async drawFaceLandmarks() { // Draw the original video frame at the canvas size this.outputCanvasCtx!.drawImage( @@ -185,49 +237,20 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface { this.outputCanvasCtx!.lineWidth = 2 for (const face of this.faceLandmarkerResult.faceLandmarks) { - // Find eye landmarks (indices 33 and 263 are the left and right eye centers) - const leftEye = face[33] - const rightEye = face[263] + // Find eye landmarks (indices 33 and 263 are the left and right eye corners) + const leftEye = face[468] + const rightEye = face[473] - if (leftEye && rightEye) { - // Calculate glasses position and size - const eyeDistance = Math.sqrt( - Math.pow(rightEye.x - leftEye.x, 2) + - Math.pow(rightEye.y - leftEye.y, 2) - ) - - // Scale glasses based on eye distance - const glassesWidth = eyeDistance * PROCESSING_WIDTH * 2.5 // Adjust multiplier as needed - const glassesHeight = glassesWidth * 0.7 // Adjust aspect ratio as needed - - // Calculate center position between eyes - const centerX = (leftEye.x + rightEye.x) / 2 - const centerY = (leftEye.y + rightEye.y) / 2 - - // Draw glasses - this.outputCanvasCtx!.save() - this.outputCanvasCtx!.translate( - centerX * PROCESSING_WIDTH, - centerY * PROCESSING_HEIGHT - ) - - // Calculate rotation angle based on eye positions - const angle = Math.atan2( - rightEye.y - leftEye.y, - rightEye.x - leftEye.x - ) - this.outputCanvasCtx!.rotate(angle) - - // Draw glasses centered at the midpoint between eyes - this.outputCanvasCtx!.drawImage( - this.glassesImage!, - -glassesWidth / 2, - -glassesHeight / 2, - glassesWidth, - glassesHeight - ) - - this.outputCanvasCtx!.restore() + // Find mouth landmarks for mustache (indices 0 and 17 are the left and right corners of the mouth) + const leftMoustache = face[92] + const rightMoustache = face[322] + + if (leftEye && rightEye && this.options.showGlasses) { + this.drawEffect(leftEye, rightEye, this.glassesImage!, 2.5, 0.7) + } + + if (leftMoustache && rightMoustache && this.options.showMustache) { + this.drawEffect(leftMoustache, rightMoustache, this.mustacheImage!, 1.5, 0.5) } } } diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index 61220587..e7f27414 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -8,7 +8,8 @@ import { FaceLandmarksProcessor } from './FaceLandmarksProcessor' export type BackgroundOptions = { blurRadius?: number imagePath?: string - showFaceLandmarks?: boolean + showGlasses?: boolean + showMustache?: boolean } export interface ProcessorSerialized { diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index 306f16c0..10e19722 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -5,17 +5,17 @@ import { BackgroundProcessorFactory, BackgroundProcessorInterface, ProcessorType, + BackgroundOptions } from '../blur' import { css } from '@/styled-system/css' import { Text, P, ToggleButton, H } from '@/primitives' import { styled } from '@/styled-system/jsx' -import { BackgroundOptions } from '@livekit/track-processors' import { BlurOn } from '@/components/icons/BlurOn' import { BlurOnStrong } from '@/components/icons/BlurOnStrong' import { useTrackToggle } from '@livekit/components-react' import { Loader } from '@/primitives/Loader' import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay' -import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react' +import { RiProhibited2Line, RiGlassesLine, RiEmotionLine } from '@remixicon/react' import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess' enum BlurRadius { @@ -141,9 +141,36 @@ export const EffectsConfiguration = ({ } const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => { + if (type === ProcessorType.FACE_LANDMARKS) { + const effect = options.showGlasses ? 'glasses' : 'mustache' + return t(`faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`) + } return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`) } + const getFaceLandmarksOptions = () => { + const processor = getProcessor() + if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) { + return processor.serialize().options as { showGlasses?: boolean; showMustache?: boolean } + } + return { showGlasses: false, showMustache: false } + } + + const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'mustache') => { + const currentOptions = getFaceLandmarksOptions() + const newOptions = { + ...currentOptions, + [effect === 'glasses' ? 'showGlasses' : 'showMustache']: !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showMustache'] + } + + if (!newOptions.showGlasses && !newOptions.showMustache) { + // If both effects are off stop the processor + await clearEffect() + } else { + await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions) + } + } + return (
- await toggleEffect(ProcessorType.FACE_LANDMARKS, { - blurRadius: 0, - }) - } - isSelected={isSelected(ProcessorType.FACE_LANDMARKS, { - blurRadius: 0, - })} - data-attr="toggle-face-landmarks" + onChange={async () => await toggleFaceLandmarkEffect('glasses')} + isSelected={getFaceLandmarksOptions().showGlasses} + data-attr="toggle-glasses" > - + + + await toggleFaceLandmarkEffect('mustache')} + isSelected={getFaceLandmarksOptions().showMustache} + data-attr="toggle-mustache" + > +
diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.json index afd2198c..2a672fd7 100644 --- a/src/frontend/src/locales/de/rooms.json +++ b/src/frontend/src/locales/de/rooms.json @@ -151,9 +151,15 @@ "clear": "Virtuellen Hintergrund deaktivieren" }, "faceLandmarks": { - "title": "Gesichtsmerkmale", - "apply": "Gesichtsmerkmale aktivieren", - "clear": "Gesichtsmerkmale deaktivieren" + "title": "Visuelle Effekte", + "glasses": { + "apply": "Brille hinzufügen", + "clear": "Brille entfernen" + }, + "mustache": { + "apply": "Schnurrbart hinzufügen", + "clear": "Schnurrbart entfernen" + } }, "experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität." }, diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.json index b94e9c70..23fc147d 100644 --- a/src/frontend/src/locales/en/rooms.json +++ b/src/frontend/src/locales/en/rooms.json @@ -150,9 +150,15 @@ "clear": "Disable virtual background" }, "faceLandmarks": { - "title": "Face landmarks", - "apply": "Enable face landmarks", - "clear": "Disable face landmarks" + "title": "Visual Effects", + "glasses": { + "apply": "Add Glasses", + "clear": "Remove Glasses" + }, + "mustache": { + "apply": "Add Mustache", + "clear": "Remove Mustache" + } }, "experimental": "Experimental feature. A v2 is coming for full browser support and improved quality." }, diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.json index 553de0c3..0508179c 100644 --- a/src/frontend/src/locales/fr/rooms.json +++ b/src/frontend/src/locales/fr/rooms.json @@ -150,9 +150,15 @@ "clear": "Désactiver l'arrière-plan virtuel" }, "faceLandmarks": { - "title": "Points du visage", - "apply": "Activer les points du visage", - "clear": "Désactiver les points du visage" + "title": "Effets visuels", + "glasses": { + "apply": "Ajouter des lunettes", + "clear": "Retirer les lunettes" + }, + "mustache": { + "apply": "Ajouter une moustache", + "clear": "Retirer la moustache" + } }, "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité." }, diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.json index 7494dde7..da56ae02 100644 --- a/src/frontend/src/locales/nl/rooms.json +++ b/src/frontend/src/locales/nl/rooms.json @@ -150,12 +150,14 @@ "clear": "Virtuele achtergrond uitschakelen" }, "faceLandmarks": { - "title": "Gezichtskenmerken", - "apply": "Gezichtskenmerken inschakelen", - "clear": "Gezichtskenmerken uitschakelen", - "tooltip": { - "apply": "Gezichtskenmerken inschakelen", - "clear": "Gezichtskenmerken uitschakelen" + "title": "Visuele effecten", + "glasses": { + "apply": "Bril toevoegen", + "clear": "Bril verwijderen" + }, + "mustache": { + "apply": "Snor toevoegen", + "clear": "Snor verwijderen" } }, "experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."