diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx
index 57c28c44..c26e4465 100644
--- a/src/frontend/src/features/rooms/components/Conference.tsx
+++ b/src/frontend/src/features/rooms/components/Conference.tsx
@@ -16,7 +16,7 @@ import { VideoConference } from '../livekit/prefabs/VideoConference'
import posthog from 'posthog-js'
import { css } from '@/styled-system/css'
import { LocalUserChoices } from '../routes/Room'
-import { BackgroundBlurFactory } from '../livekit/components/blur'
+import { BackgroundProcessorFactory } from '../livekit/components/blur'
export const Conference = ({
roomId,
@@ -114,7 +114,7 @@ export const Conference = ({
audio={userConfig.audioEnabled}
video={
userConfig.videoEnabled && {
- processor: BackgroundBlurFactory.deserializeProcessor(
+ processor: BackgroundProcessorFactory.deserializeProcessor(
userConfig.processorSerialized
),
}
diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsx
index 4a942545..61cbfd39 100644
--- a/src/frontend/src/features/rooms/components/Join.tsx
+++ b/src/frontend/src/features/rooms/components/Join.tsx
@@ -7,9 +7,8 @@ import { LocalVideoTrack, Track } from 'livekit-client'
import { H } from '@/primitives/H'
import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice'
import { Field } from '@/primitives/Field'
-import { Form } from '@/primitives'
+import { Button, Dialog, Text, Form } from '@/primitives'
import { HStack, VStack } from '@/styled-system/jsx'
-import { Button, Dialog } from '@/primitives'
import { LocalUserChoices } from '../routes/Room'
import { Heading } from 'react-aria-components'
import { RiImageCircleAiFill } from '@remixicon/react'
@@ -18,7 +17,7 @@ import {
EffectsConfigurationProps,
} from '../livekit/components/effects/EffectsConfiguration'
import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
-import { BackgroundBlurFactory } from '../livekit/components/blur'
+import { BackgroundProcessorFactory } from '../livekit/components/blur'
import { isMobileBrowser } from '@livekit/components-core'
const onError = (e: Error) => console.error('ERROR', e)
@@ -31,7 +30,7 @@ const Effects = ({
const [isDialogOpen, setIsDialogOpen] = useState(false)
const openDialog = () => setIsDialogOpen(true)
- if (!BackgroundBlurFactory.isSupported() || isMobileBrowser()) {
+ if (!BackgroundProcessorFactory.isSupported() || isMobileBrowser()) {
return
}
@@ -49,11 +48,19 @@ const Effects = ({
level={1}
className={css({
textStyle: 'h1',
- marginBottom: '1.5rem',
+ marginBottom: '0.25rem',
})}
>
{t('title')}
+
+ {t('subTitle')}
+
(
initialUserChoices.audioDeviceId
)
@@ -113,7 +122,7 @@ export const Join = ({
)
const [username, setUsername] = useState(initialUserChoices.username)
const [processor, setProcessor] = useState(
- BackgroundBlurFactory.deserializeProcessor(
+ BackgroundProcessorFactory.deserializeProcessor(
initialUserChoices.processorSerialized
)
)
@@ -129,12 +138,15 @@ export const Join = ({
useEffect(() => {
saveUsername(username)
}, [username, saveUsername])
+
useEffect(() => {
saveProcessorSerialized(processor?.serialize())
- }, [processor, saveProcessorSerialized])
-
- const [audioEnabled, setAudioEnabled] = useState(true)
- const [videoEnabled, setVideoEnabled] = useState(true)
+ }, [
+ processor,
+ saveProcessorSerialized,
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ JSON.stringify(processor?.serialize()),
+ ])
const tracks = usePreviewTracks(
{
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts
index 5481a9ff..9d362c7f 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts
@@ -5,7 +5,7 @@ import {
} from '@livekit/track-processors'
import { ProcessorOptions, Track } from 'livekit-client'
import {
- BackgroundBlurProcessorInterface,
+ BackgroundProcessorInterface,
BackgroundOptions,
ProcessorType,
} from '.'
@@ -16,9 +16,9 @@ import {
* used accross the project.
*/
export class BackgroundBlurTrackProcessorJsWrapper
- implements BackgroundBlurProcessorInterface
+ implements BackgroundProcessorInterface
{
- name: string = 'Blur'
+ name: string = 'blur'
processor: ProcessorWrapper
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
similarity index 76%
rename from src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts
rename to src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
index 781e0d66..744a749f 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
@@ -12,7 +12,7 @@ import {
timerWorkerScript,
} from './TimerWorker'
import {
- BackgroundBlurProcessorInterface,
+ BackgroundProcessorInterface,
BackgroundOptions,
ProcessorType,
} from '.'
@@ -32,9 +32,7 @@ const DEFAULT_BLUR = '10'
* It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and
* MediaStreamTrackProcessor.
*/
-export class BackgroundBlurCustomProcessor
- implements BackgroundBlurProcessorInterface
-{
+export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
options: BackgroundOptions
name: string
processedTrack?: MediaStreamTrack | undefined
@@ -63,9 +61,18 @@ export class BackgroundBlurCustomProcessor
timerWorker?: Worker
+ type: ProcessorType
+ virtualBackgroundImage?: HTMLImageElement
+
constructor(opts: BackgroundOptions) {
this.name = 'blur'
this.options = opts
+
+ if (this.options.blurRadius) {
+ this.type = ProcessorType.BLUR
+ } else {
+ this.type = ProcessorType.VIRTUAL
+ }
}
static get isSupported() {
@@ -81,6 +88,7 @@ export class BackgroundBlurCustomProcessor
this.sourceSettings = this.source!.getSettings()
this.videoElement = opts.element as HTMLVideoElement
+ this._initVirtualBackgroundImage()
this._createMainCanvas()
this._createMaskCanvas()
@@ -98,8 +106,21 @@ export class BackgroundBlurCustomProcessor
posthog.capture('firefox-blurring-init')
}
+ _initVirtualBackgroundImage() {
+ const needsUpdate =
+ this.options.imagePath &&
+ this.virtualBackgroundImage &&
+ this.virtualBackgroundImage.src !== this.options.imagePath
+ if (this.options.imagePath || needsUpdate) {
+ this.virtualBackgroundImage = document.createElement('img')
+ this.virtualBackgroundImage.crossOrigin = 'anonymous'
+ this.virtualBackgroundImage.src = this.options.imagePath!
+ }
+ }
+
update(opts: BackgroundOptions): void {
this.options = opts
+ this._initVirtualBackgroundImage()
}
_initWorker() {
@@ -201,6 +222,7 @@ export class BackgroundBlurCustomProcessor
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
this.outputCanvasCtx!.filter = 'blur(8px)'
+ // Put opacity mask.
this.outputCanvasCtx!.drawImage(
this.segmentationMaskCanvas!,
0,
@@ -213,19 +235,69 @@ export class BackgroundBlurCustomProcessor
this.videoElement!.videoHeight
)
+ // Draw clear body.
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+ // Draw blurry background.
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
}
+ /**
+ * TODO: future improvement with WebGL.
+ */
+ async drawVirtualBackground() {
+ const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
+ for (let i = 0; i < mask.length; ++i) {
+ this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
+ }
+
+ this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
+
+ this.outputCanvasCtx!.globalCompositeOperation = 'copy'
+ this.outputCanvasCtx!.filter = 'blur(8px)'
+
+ // Put opacity mask.
+ this.outputCanvasCtx!.drawImage(
+ this.segmentationMaskCanvas!,
+ 0,
+ 0,
+ PROCESSING_WIDTH,
+ PROCESSING_HEIGHT,
+ 0,
+ 0,
+ this.videoElement!.videoWidth,
+ this.videoElement!.videoHeight
+ )
+
+ // Draw clear body.
+ this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
+ this.outputCanvasCtx!.filter = 'none'
+ this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+
+ // Draw virtual background.
+ this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
+ this.outputCanvasCtx!.drawImage(
+ this.virtualBackgroundImage!,
+ 0,
+ 0,
+ this.outputCanvas!.width,
+ this.outputCanvas!.height
+ )
+ }
+
async process() {
await this.sizeSource()
await this.segment()
- await this.blur()
+
+ if (this.options.blurRadius) {
+ await this.blur()
+ } else {
+ await this.drawVirtualBackground()
+ }
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
@@ -284,12 +356,12 @@ export class BackgroundBlurCustomProcessor
}
clone() {
- return new BackgroundBlurCustomProcessor(this.options)
+ return new BackgroundCustomProcessor(this.options)
}
serialize() {
return {
- type: ProcessorType.BLUR,
+ type: this.type,
options: this.options,
}
}
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
index eaa4a372..893b9eb0 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -1,10 +1,12 @@
import { ProcessorWrapper } from '@livekit/track-processors'
import { Track, TrackProcessor } from 'livekit-client'
import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
-import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor'
+import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
+import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'
export type BackgroundOptions = {
blurRadius?: number
+ imagePath?: string
}
export interface ProcessorSerialized {
@@ -12,40 +14,49 @@ export interface ProcessorSerialized {
options: BackgroundOptions
}
-export interface BackgroundBlurProcessorInterface
+export interface BackgroundProcessorInterface
extends TrackProcessor {
update(opts: BackgroundOptions): void
options: BackgroundOptions
- clone(): BackgroundBlurProcessorInterface
+ clone(): BackgroundProcessorInterface
serialize(): ProcessorSerialized
}
export enum ProcessorType {
BLUR = 'blur',
+ VIRTUAL = 'virtual',
}
-export class BackgroundBlurFactory {
+export class BackgroundProcessorFactory {
static isSupported() {
- return (
- ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported
- )
+ return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported
}
static getProcessor(
+ type: ProcessorType,
opts: BackgroundOptions
- ): BackgroundBlurProcessorInterface | undefined {
- if (ProcessorWrapper.isSupported) {
- return new BackgroundBlurTrackProcessorJsWrapper(opts)
- }
- if (BackgroundBlurCustomProcessor.isSupported) {
- return new BackgroundBlurCustomProcessor(opts)
+ ): BackgroundProcessorInterface | undefined {
+ if (type === ProcessorType.BLUR) {
+ if (ProcessorWrapper.isSupported) {
+ return new BackgroundBlurTrackProcessorJsWrapper(opts)
+ }
+ if (BackgroundCustomProcessor.isSupported) {
+ return new BackgroundCustomProcessor(opts)
+ }
+ } else if (type === ProcessorType.VIRTUAL) {
+ if (ProcessorWrapper.isSupported) {
+ return new BackgroundVirtualTrackProcessorJsWrapper(opts)
+ }
+ if (BackgroundCustomProcessor.isSupported) {
+ return new BackgroundCustomProcessor(opts)
+ }
}
return undefined
}
static deserializeProcessor(data?: ProcessorSerialized) {
- if (data?.type === ProcessorType.BLUR) {
- return BackgroundBlurFactory.getProcessor(data?.options)
+ if (data?.type) {
+ return BackgroundProcessorFactory.getProcessor(data?.type, data?.options)
}
return undefined
}
diff --git a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx
index efb6a91a..46dd442c 100644
--- a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx
@@ -27,7 +27,7 @@ import { css } from '@/styled-system/css'
import { ButtonRecipeProps } from '@/primitives/buttonRecipe'
import { useEffect } from 'react'
import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
-import { BackgroundBlurFactory } from '../blur'
+import { BackgroundProcessorFactory } from '../blur'
export type ToggleSource = Exclude<
Track.Source,
@@ -115,7 +115,7 @@ export const SelectToggleDevice = ({
*
* See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121
*/
- const processor = BackgroundBlurFactory.deserializeProcessor(
+ const processor = BackgroundProcessorFactory.deserializeProcessor(
userChoices.processorSerialized
)
diff --git a/src/frontend/src/primitives/Text.tsx b/src/frontend/src/primitives/Text.tsx
index 37c2d576..15231f2a 100644
--- a/src/frontend/src/primitives/Text.tsx
+++ b/src/frontend/src/primitives/Text.tsx
@@ -28,6 +28,14 @@ export const text = cva({
paddingTop: 'heading',
},
},
+ subTitle: {
+ fontSize: '1rem',
+ color: 'greyscale.600',
+ },
+ bodyXsBold: {
+ textStyle: 'body',
+ fontWeight: 'bold',
+ },
body: {
textStyle: 'body',
},