From 56379f2d6ee215f2b2a63dffd988242207353fa8 Mon Sep 17 00:00:00 2001 From: Nathan Vasse Date: Tue, 7 Jan 2025 17:01:20 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(front)=20add=20blurring=20support=20o?= =?UTF-8?q?n=20Firefox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We cannot use track-processor-js for Firefox because it uses various approaches not compatible with Firefox. Firefox cannot run inference on GPU, so we must use CPU, we could also not use MediaStreamTrackProcessor nor MediaStreamTrackGenerator. So I had to make a new implementation from the ground up, using canvas filters and CPU inferences. --- .../rooms/livekit/components/Effects.tsx | 25 +- .../blur/BackgroundBlurCustomProcessor.ts | 278 ++++++++++++++++++ .../BackgroundBlurTrackProcessorJsWrapper.ts | 48 +++ .../livekit/components/blur/TimerWorker.ts | 71 +++++ .../rooms/livekit/components/blur/index.ts | 32 ++ src/frontend/src/locales/en/rooms.json | 2 +- src/frontend/src/locales/fr/rooms.json | 2 +- 7 files changed, 443 insertions(+), 15 deletions(-) create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/TimerWorker.ts create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/index.ts diff --git a/src/frontend/src/features/rooms/livekit/components/Effects.tsx b/src/frontend/src/features/rooms/livekit/components/Effects.tsx index afece6f6..bbdc9a0b 100644 --- a/src/frontend/src/features/rooms/livekit/components/Effects.tsx +++ b/src/frontend/src/features/rooms/livekit/components/Effects.tsx @@ -5,11 +5,9 @@ import { Text, P, ToggleButton, Div, H } from '@/primitives' import { useTranslation } from 'react-i18next' import { HStack, styled, VStack } from '@/styled-system/jsx' import { - BackgroundBlur, - BackgroundOptions, - ProcessorWrapper, - BackgroundTransformer, -} from '@livekit/track-processors' + BackgroundBlurFactory, + BackgroundBlurProcessorInterface, +} from './blur/index' const Information = styled('div', { base: { @@ -27,6 +25,8 @@ enum BlurRadius { NORMAL = 10, } +const isSupported = BackgroundBlurFactory.isSupported() + export const Effects = () => { const { t } = useTranslation('rooms', { keyPrefix: 'effects' }) const { isCameraEnabled, cameraTrack, localParticipant } = @@ -37,15 +37,12 @@ export const Effects = () => { const localCameraTrack = cameraTrack?.track as LocalVideoTrack const getProcessor = () => { - return localCameraTrack?.getProcessor() as ProcessorWrapper + return localCameraTrack?.getProcessor() as BackgroundBlurProcessorInterface } const getBlurRadius = (): BlurRadius => { const processor = getProcessor() - return ( - (processor?.transformer as BackgroundTransformer)?.options?.blurRadius || - BlurRadius.NONE - ) + return processor?.options.blurRadius || BlurRadius.NONE } const toggleBlur = async (blurRadius: number) => { @@ -61,9 +58,11 @@ export const Effects = () => { if (blurRadius == currentBlurRadius && processor) { await localCameraTrack.stopProcessor() } else if (!processor) { - await localCameraTrack.setProcessor(BackgroundBlur(blurRadius)) + await localCameraTrack.setProcessor( + BackgroundBlurFactory.getProcessor({ blurRadius })! + ) } else { - await processor?.updateTransformerOptions({ blurRadius }) + processor?.update({ blurRadius }) } } catch (error) { console.error('Error applying blur:', error) @@ -147,7 +146,7 @@ export const Effects = () => { > {t('heading')} - {ProcessorWrapper.isSupported ? ( + {isSupported ? ( ) { + if (!opts.element) { + throw new Error('Element is required for processing') + } + + this.source = opts.track as MediaStreamTrack + this.sourceSettings = this.source!.getSettings() + this.videoElement = opts.element as HTMLVideoElement + + this._createMainCanvas() + this._createMaskCanvas() + + const stream = this.outputCanvas!.captureStream() + const tracks = stream.getVideoTracks() + if (tracks.length == 0) { + throw new Error('No tracks found for processing') + } + this.processedTrack = tracks[0] + + this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT) + await this.initSegmenter() + this._initWorker() + } + + update(opts: BackgroundOptions): void { + this.options = opts + } + + _initWorker() { + this.timerWorker = new Worker(timerWorkerScript, { + name: 'Blurring', + }) + this.timerWorker.onmessage = (data) => this.onTimerMessage(data) + // When hiding camera then showing it again, the onloadeddata callback is not fired again. + if (this.videoElementLoaded) { + this.timerWorker!.postMessage({ + id: SET_TIMEOUT, + timeMs: 1000 / 30, + }) + } else { + this.videoElement!.onloadeddata = () => { + this.videoElementLoaded = true + this.timerWorker!.postMessage({ + id: SET_TIMEOUT, + timeMs: 1000 / 30, + }) + } + } + } + + onTimerMessage(response: { data: { id: number } }) { + if (response.data.id === TIMEOUT_TICK) { + this.process() + } + } + + async initSegmenter() { + const vision = await FilesetResolver.forVisionTasks( + 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm' + ) + this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, { + baseOptions: { + modelAssetPath: + 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', + delegate: 'CPU', // Use CPU for Firefox. + }, + runningMode: 'VIDEO', + outputCategoryMask: true, + outputConfidenceMasks: false, + }) + } + + /** + * Resize the source video to the processing resolution. + */ + async sizeSource() { + this.segmentationMaskCanvasCtx?.drawImage( + this.videoElement!, + 0, + 0, + this.videoElement!.videoWidth, + this.videoElement!.videoHeight, + 0, + 0, + PROCESSING_WIDTH, + PROCESSING_HEIGHT + ) + + this.sourceImageData = this.segmentationMaskCanvasCtx?.getImageData( + 0, + 0, + PROCESSING_WIDTH, + PROCESSING_WIDTH + ) + } + + /** + * Run the segmentation. + */ + async segment() { + const startTimeMs = performance.now() + return new Promise((resolve) => { + this.imageSegmenter!.segmentForVideo( + this.sourceImageData!, + startTimeMs, + (result: ImageSegmenterResult) => { + this.imageSegmenterResult = result + resolve() + } + ) + }) + } + + /** + * TODO: future improvement with WebGL. + */ + async blur() { + const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() + for (let i = 0; i < mask.length; ++i) { + this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] + } + + this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) + + this.outputCanvasCtx!.globalCompositeOperation = 'copy' + this.outputCanvasCtx!.filter = 'blur(8px)' + + this.outputCanvasCtx!.drawImage( + this.segmentationMaskCanvas!, + 0, + 0, + PROCESSING_WIDTH, + PROCESSING_HEIGHT, + 0, + 0, + this.videoElement!.videoWidth, + this.videoElement!.videoHeight + ) + + this.outputCanvasCtx!.globalCompositeOperation = 'source-in' + this.outputCanvasCtx!.filter = 'none' + this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + + this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' + this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)` + this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) + } + + async process() { + await this.sizeSource() + await this.segment() + await this.blur() + this.timerWorker!.postMessage({ + id: SET_TIMEOUT, + timeMs: 1000 / 30, + }) + } + + _createMainCanvas() { + this.outputCanvas = document.querySelector( + 'canvas#background-blur-local' + ) as HTMLCanvasElement + if (!this.outputCanvas) { + this.outputCanvas = this._createCanvas( + BLUR_CANVAS_ID, + this.sourceSettings!.width!, + this.sourceSettings!.height! + ) + } + this.outputCanvasCtx = this.outputCanvas.getContext('2d')! + } + + _createMaskCanvas() { + this.segmentationMaskCanvas = document.querySelector( + `#${SEGMENTATION_MASK_CANVAS_ID}` + ) as HTMLCanvasElement + if (!this.segmentationMaskCanvas) { + this.segmentationMaskCanvas = this._createCanvas( + SEGMENTATION_MASK_CANVAS_ID, + PROCESSING_WIDTH, + PROCESSING_HEIGHT + ) + } + this.segmentationMaskCanvasCtx = + this.segmentationMaskCanvas.getContext('2d')! + } + + _createCanvas(id: string, width: number, height: number) { + const element = document.createElement('canvas') + element.setAttribute('id', id) + element.setAttribute('width', '' + width) + element.setAttribute('height', '' + height) + return element + } + + async restart(opts: ProcessorOptions) { + await this.destroy() + return this.init(opts) + } + + async destroy() { + this.timerWorker?.postMessage({ + id: CLEAR_TIMEOUT, + }) + + this.timerWorker?.terminate() + this.imageSegmenter?.close() + } +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts new file mode 100644 index 00000000..e409074c --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts @@ -0,0 +1,48 @@ +import { + BackgroundBlur, + BackgroundTransformer, + ProcessorWrapper, +} from '@livekit/track-processors' +import { ProcessorOptions, Track } from 'livekit-client' +import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.' + +/** + * This is simply a wrapper around track-processor-js Processor + * in order to be compatible with a common interface BackgroundBlurProcessorInterface + * used accross the project. + */ +export class BackgroundBlurTrackProcessorJsWrapper + implements BackgroundBlurProcessorInterface +{ + name: string = 'Blur' + + processor: ProcessorWrapper + + constructor(opts: BackgroundOptions) { + this.processor = BackgroundBlur(opts.blurRadius) + } + + async init(opts: ProcessorOptions) { + return this.processor.init(opts) + } + + async restart(opts: ProcessorOptions) { + return this.processor.restart(opts) + } + + async destroy() { + return this.processor.destroy() + } + + update(opts: BackgroundOptions): void { + this.processor.updateTransformerOptions(opts) + } + + get processedTrack() { + return this.processor.processedTrack + } + + get options() { + return (this.processor.transformer as BackgroundTransformer).options + } +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/TimerWorker.ts b/src/frontend/src/features/rooms/livekit/components/blur/TimerWorker.ts new file mode 100644 index 00000000..ac928084 --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/TimerWorker.ts @@ -0,0 +1,71 @@ +/** + * From https://github.com/jitsi/jitsi-meet + */ + +/** + * SET_TIMEOUT constant is used to set interval and it is set in + * the id property of the request.data property. TimeMs property must + * also be set. + * + * ``` + * //Request.data example: + * { + * id: SET_TIMEOUT, + * timeMs: 33 + * } + * ``` + */ +export const SET_TIMEOUT = 1 + +/** + * CLEAR_TIMEOUT constant is used to clear the interval and it is set in + * the id property of the request.data property. + * + * ``` + * { + * id: CLEAR_TIMEOUT + * } + * ``` + */ +export const CLEAR_TIMEOUT = 2 + +/** + * TIMEOUT_TICK constant is used as response and it is set in the id property. + * + * ``` + * { + * id: TIMEOUT_TICK + * } + * ``` + */ +export const TIMEOUT_TICK = 3 + +/** + * The following code is needed as string to create a URL from a Blob. + * The URL is then passed to a WebWorker. Reason for this is to enable + * use of setInterval that is not throttled when tab is inactive. + */ +const code = ` + var timer; + + onmessage = function(request) { + switch (request.data.id) { + case ${SET_TIMEOUT}: { + timer = setTimeout(() => { + postMessage({ id: ${TIMEOUT_TICK} }); + }, request.data.timeMs); + break; + } + case ${CLEAR_TIMEOUT}: { + if (timer) { + clearTimeout(timer); + } + break; + } + } + }; +` + +export const timerWorkerScript = URL.createObjectURL( + new Blob([code], { type: 'application/javascript' }) +) diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts new file mode 100644 index 00000000..7ab10dbb --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -0,0 +1,32 @@ +import { ProcessorWrapper } from '@livekit/track-processors' +import { Track, TrackProcessor } from 'livekit-client' +import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper' +import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor' + +export type BackgroundOptions = { + blurRadius?: number +} + +export interface BackgroundBlurProcessorInterface + extends TrackProcessor { + update(opts: BackgroundOptions): void + options: BackgroundOptions +} + +export class BackgroundBlurFactory { + static isSupported() { + return ( + ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported + ) + } + + static getProcessor(opts: BackgroundOptions) { + if (ProcessorWrapper.isSupported) { + return new BackgroundBlurTrackProcessorJsWrapper(opts) + } + if (BackgroundBlurCustomProcessor.isSupported) { + return new BackgroundBlurCustomProcessor(opts) + } + return null + } +} diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.json index 1f618510..a1fa8f2e 100644 --- a/src/frontend/src/locales/en/rooms.json +++ b/src/frontend/src/locales/en/rooms.json @@ -87,7 +87,7 @@ }, "effects": { "activateCamera": "Your camera is disabled. Choose an option to enable it.", - "notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome :(", + "notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome for best performance or Firefox :(", "heading": "Blur", "blur": { "light": "Light blur", diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.json index 872d4c7c..92bf45fe 100644 --- a/src/frontend/src/locales/fr/rooms.json +++ b/src/frontend/src/locales/fr/rooms.json @@ -87,7 +87,7 @@ }, "effects": { "activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.", - "notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome :(", + "notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour une meilleure performance ou Firefox :(", "heading": "Flou", "blur": { "light": "Léger flou",