From b27c5e9b926bc871c4bd9112bb221f6cdd992de9 Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Wed, 30 Apr 2025 17:23:50 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F(frontend)=20decouple=20la?= =?UTF-8?q?ndmark=20processor=20from=20background=20processors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate landmark processor logic to avoid entanglement with background processing. Ensures future refactoring can replace custom background implementation without affecting landmark functionality. --- .../components/blur/FaceLandmarksProcessor.ts | 21 ++- .../rooms/livekit/components/blur/index.ts | 13 +- .../effects/EffectsConfiguration.tsx | 177 +++++------------- .../components/effects/FunnyEffects.tsx | 120 ++++++++++++ .../features/rooms/livekit/hooks/useKonami.ts | 0 5 files changed, 175 insertions(+), 156 deletions(-) create mode 100644 src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx create mode 100644 src/frontend/src/features/rooms/livekit/hooks/useKonami.ts diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts index 4084072c..7e59d334 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts @@ -1,4 +1,4 @@ -import { ProcessorOptions, Track } from 'livekit-client' +import { ProcessorOptions, Track, TrackProcessor } from 'livekit-client' import posthog from 'posthog-js' import { FilesetResolver, @@ -11,19 +11,20 @@ import { TIMEOUT_TICK, timerWorkerScript, } from './TimerWorker' -import { - BackgroundProcessorInterface, - BackgroundOptions, - ProcessorType, -} from '.' +import { ProcessorType } from '.' const PROCESSING_WIDTH = 256 * 3 const PROCESSING_HEIGHT = 144 * 3 const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local' -export class FaceLandmarksProcessor implements BackgroundProcessorInterface { - options: BackgroundOptions +export type FaceLandmarksOptions = { + showGlasses: boolean + showFrench: boolean +} + +export class FaceLandmarksProcessor implements TrackProcessor { + options: FaceLandmarksOptions name: string processedTrack?: MediaStreamTrack | undefined @@ -50,7 +51,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface { glassesImage?: HTMLImageElement mustacheImage?: HTMLImageElement beretImage?: HTMLImageElement - constructor(opts: BackgroundOptions) { + constructor(opts: FaceLandmarksOptions) { this.name = 'face_landmarks' this.options = opts this.type = ProcessorType.FACE_LANDMARKS @@ -314,7 +315,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface { return element } - update(opts: BackgroundOptions): void { + update(opts: FaceLandmarksOptions): void { this.options = opts } diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index 4700c520..3d59ba84 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -3,13 +3,10 @@ import { Track, TrackProcessor } from 'livekit-client' import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper' import { BackgroundCustomProcessor } from './BackgroundCustomProcessor' import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper' -import { FaceLandmarksProcessor } from './FaceLandmarksProcessor' export type BackgroundOptions = { blurRadius?: number imagePath?: string - showGlasses?: boolean - showFrench?: boolean } export interface ProcessorSerialized { @@ -37,11 +34,7 @@ export class BackgroundProcessorFactory { } static isSupported() { - return ( - ProcessorWrapper.isSupported || - BackgroundCustomProcessor.isSupported || - FaceLandmarksProcessor.isSupported - ) + return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported } static getProcessor( @@ -62,10 +55,6 @@ export class BackgroundProcessorFactory { if (BackgroundCustomProcessor.isSupported) { return new BackgroundCustomProcessor(opts) } - } else if (type === ProcessorType.FACE_LANDMARKS) { - if (FaceLandmarksProcessor.isSupported) { - return new FaceLandmarksProcessor(opts) - } } return undefined } diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index 5d7c8c0b..a2ef25c1 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -15,11 +15,8 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong' import { useTrackToggle } from '@livekit/components-react' import { Loader } from '@/primitives/Loader' import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay' -import { - RiProhibited2Line, - RiGlassesLine, - RiGoblet2Fill, -} from '@remixicon/react' +import { RiProhibited2Line } from '@remixicon/react' +import { FunnyEffects } from './FunnyEffects' import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess' enum BlurRadius { @@ -152,42 +149,9 @@ export const EffectsConfiguration = ({ } const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => { - if (type === ProcessorType.FACE_LANDMARKS) { - const effect = options.showGlasses ? 'glasses' : 'french' - return t( - `faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}` - ) - } return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`) } - const getFaceLandmarksOptions = () => { - const processor = getProcessor() - if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) { - return processor.serialize().options as { - showGlasses?: boolean - showFrench?: boolean - } - } - return { showGlasses: false, showFrench: false } - } - - const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'french') => { - const currentOptions = getFaceLandmarksOptions() - const newOptions = { - ...currentOptions, - [effect === 'glasses' ? 'showGlasses' : 'showFrench']: - !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showFrench'], - } - - if (!newOptions.showGlasses && !newOptions.showFrench) { - // If both effects are off stop the processor - await clearEffect() - } else { - await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions) - } - } - return (
+ {hasFaceLandmarksAccess && ( + + )} {isSupported ? ( <>
@@ -347,8 +318,6 @@ export const EffectsConfiguration = ({
-
- {hasFaceLandmarksAccess && (
- {t('faceLandmarks.title')} + {t('virtual.title')}
- - await toggleFaceLandmarkEffect('glasses') - } - isSelected={getFaceLandmarksOptions().showGlasses} - data-attr="toggle-glasses" - > - - - - await toggleFaceLandmarkEffect('french') - } - isSelected={getFaceLandmarksOptions().showFrench} - data-attr="toggle-french" - > - - -
-
- )} -
- - {t('virtual.title')} - -
- {[...Array(8).keys()].map((i) => { - const imagePath = `/assets/backgrounds/${i + 1}.jpg` - const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg` - return ( - - await toggleEffect(ProcessorType.VIRTUAL, { + {[...Array(8).keys()].map((i) => { + const imagePath = `/assets/backgrounds/${i + 1}.jpg` + const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg` + return ( + - ) - })} + })} + tooltip={tooltipLabel(ProcessorType.VIRTUAL, { + imagePath, + })} + isDisabled={processorPendingReveal} + onChange={async () => + await toggleEffect(ProcessorType.VIRTUAL, { + imagePath, + }) + } + isSelected={isSelected(ProcessorType.VIRTUAL, { + imagePath, + })} + className={css({ + bgSize: 'cover', + })} + style={{ + backgroundImage: `url(${thumbnailPath})`, + }} + data-attr={`toggle-virtual-${i}`} + /> + ) + })} +
diff --git a/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx new file mode 100644 index 00000000..c19eb43a --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx @@ -0,0 +1,120 @@ +import { css } from '@/styled-system/css' +import { H, ToggleButton } from '@/primitives' +import { ProcessorType } from '../blur' +import { RiGlassesLine, RiGoblet2Fill } from '@remixicon/react' +import { useTranslation } from 'react-i18next' +import { FaceLandmarksProcessor } from '../blur/FaceLandmarksProcessor' +import { LocalVideoTrack } from 'livekit-client' + +export type FunnyEffectsProps = { + videoTrack: LocalVideoTrack + isPending?: boolean + onPending: (value: boolean) => void +} + +export const FunnyEffects = ({ + videoTrack, + isPending, + onPending, +}: FunnyEffectsProps) => { + const { t } = useTranslation('rooms', { keyPrefix: 'effects' }) + + const getOptions = () => { + const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor + if (!processor || processor.type != ProcessorType.FACE_LANDMARKS) { + return { + showGlasses: false, + showFrench: false, + } + } + return processor.serialize().options + } + + const options = getOptions() + + const toggleFaceLandmarkEffect = async ( + showEffect: 'showGlasses' | 'showFrench' + ) => { + const options = getOptions() + const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor + + const newOptions = { + ...options, + [showEffect]: !options[showEffect], + } + + onPending(true) + + try { + if (!newOptions.showGlasses && !newOptions.showFrench) { + await videoTrack.stopProcessor() + } else if (options.showGlasses || options.showFrench) { + await processor?.update(newOptions) + } else { + const newProcessor = new FaceLandmarksProcessor(newOptions) + await videoTrack.setProcessor(newProcessor) + } + } catch (e) { + console.error('could not update processor', e) + } finally { + onPending(false) + } + } + + const getLabelAction = (enabled: boolean) => (enabled ? 'clear' : 'apply') + + return ( +
+ + {t('faceLandmarks.title')} + +
+ await toggleFaceLandmarkEffect('showGlasses')} + isSelected={options.showGlasses} + data-attr="toggle-glasses" + > + + + await toggleFaceLandmarkEffect('showFrench')} + isSelected={options.showFrench} + data-attr="toggle-french" + > + + +
+
+ ) +} diff --git a/src/frontend/src/features/rooms/livekit/hooks/useKonami.ts b/src/frontend/src/features/rooms/livekit/hooks/useKonami.ts new file mode 100644 index 00000000..e69de29b