✨(front) add blurring feature on join

This adds a button that opens a modal that allow user to enable
video effects on join screen.

src/frontend/src/features/home/routes/Home.tsx

@@ -9,7 +9,6 @@ import { useUser, UserAware } from '@/features/auth'
 import { JoinMeetingDialog } from '../components/JoinMeetingDialog'
 import { ProConnectButton } from '@/components/ProConnectButton'
 import { useCreateRoom } from '@/features/rooms'
-import { usePersistentUserChoices } from '@livekit/components-react'
 import { RiAddLine, RiLink } from '@remixicon/react'
 import { LaterMeetingDialog } from '@/features/home/components/LaterMeetingDialog'
 import { IntroSlider } from '@/features/home/components/IntroSlider'
@@ -18,6 +17,7 @@ import { ReactNode, useState } from 'react'
 
 import { css } from '@/styled-system/css'
 import { menuRecipe } from '@/primitives/menuRecipe.ts'
+import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices'
 
 const Columns = ({ children }: { children?: ReactNode }) => {
   return (

src/frontend/src/features/rooms/components/Conference.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useMemo, useState } from 'react'
 import { useQuery } from '@tanstack/react-query'
 import { useTranslation } from 'react-i18next'
-import { LiveKitRoom, type LocalUserChoices } from '@livekit/components-react'
+import { LiveKitRoom } from '@livekit/components-react'
 import { Room, RoomOptions } from 'livekit-client'
 import { keys } from '@/api/queryKeys'
 import { queryClient } from '@/api/queryClient'
@@ -12,10 +12,11 @@ import { fetchRoom } from '../api/fetchRoom'
 import { ApiRoom } from '../api/ApiRoom'
 import { useCreateRoom } from '../api/createRoom'
 import { InviteDialog } from './InviteDialog'
-
 import { VideoConference } from '../livekit/prefabs/VideoConference'
 import posthog from 'posthog-js'
 import { css } from '@/styled-system/css'
+import { LocalUserChoices } from '../routes/Room'
+import { BackgroundBlurFactory } from '../livekit/components/blur'
 
 export const Conference = ({
   roomId,
@@ -111,7 +112,13 @@ export const Conference = ({
           token={data?.livekit?.token}
           connect={true}
           audio={userConfig.audioEnabled}
-          video={userConfig.videoEnabled}
+          video={
+            userConfig.videoEnabled && {
+              processor: BackgroundBlurFactory.deserializeProcessor(
+                userConfig.processorSerialized
+              ),
+            }
+          }
           connectOptions={connectOptions}
           className={css({
             backgroundColor: 'primaryDark.50 !important',

src/frontend/src/features/rooms/components/Join.tsx

@@ -1,9 +1,5 @@
 import { useTranslation } from 'react-i18next'
-import {
-  usePersistentUserChoices,
-  usePreviewTracks,
-  type LocalUserChoices,
-} from '@livekit/components-react'
+import { usePreviewTracks } from '@livekit/components-react'
 import { css } from '@/styled-system/css'
 import { Screen } from '@/layout/Screen'
 import { useMemo, useEffect, useRef, useState } from 'react'
@@ -13,6 +9,13 @@ import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleD
 import { Field } from '@/primitives/Field'
 import { Form } from '@/primitives'
 import { HStack, VStack } from '@/styled-system/jsx'
+import { Button, Dialog } from '@/primitives'
+import { LocalUserChoices } from '../routes/Room'
+import { Heading } from 'react-aria-components'
+import { RiImageCircleAiFill } from '@remixicon/react'
+import { EffectsConfiguration } from '../livekit/components/effects/EffectsConfiguration'
+import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
+import { BackgroundBlurFactory } from '../livekit/components/blur'
 
 const onError = (e: Error) => console.error('ERROR', e)
 
@@ -28,6 +31,7 @@ export const Join = ({
     saveAudioInputDeviceId,
     saveVideoInputDeviceId,
     saveUsername,
+    saveProcessorSerialized,
   } = usePersistentUserChoices({})
 
   const [audioDeviceId, setAudioDeviceId] = useState<string>(
@@ -37,6 +41,11 @@ export const Join = ({
     initialUserChoices.videoDeviceId
   )
   const [username, setUsername] = useState<string>(initialUserChoices.username)
+  const [processor, setProcessor] = useState(
+    BackgroundBlurFactory.deserializeProcessor(
+      initialUserChoices.processorSerialized
+    )
+  )
 
   useEffect(() => {
     saveAudioInputDeviceId(audioDeviceId)
@@ -49,6 +58,9 @@ export const Join = ({
   useEffect(() => {
     saveUsername(username)
   }, [username, saveUsername])
+  useEffect(() => {
+    saveProcessorSerialized(processor?.serialize())
+  }, [processor, saveProcessorSerialized])
 
   const [audioEnabled, setAudioEnabled] = useState(true)
   const [videoEnabled, setVideoEnabled] = useState(true)
@@ -107,11 +119,52 @@ export const Join = ({
       audioDeviceId,
       videoDeviceId,
       username,
+      processorSerialized: processor?.serialize(),
     })
   }
 
+  const [isEffectsOpen, setEffectsOpen] = useState(false)
+
+  const openEffects = () => {
+    setEffectsOpen(true)
+  }
+
+  // This hook is used to setup the persisted user choice processor on initialization.
+  // So it's on purpose that processor is not included in the deps.
+  // We just want to wait for the videoTrack to be loaded to apply the default processor.
+  useEffect(() => {
+    if (processor && videoTrack && !videoTrack.getProcessor()) {
+      videoTrack.setProcessor(processor)
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [videoTrack])
+
   return (
     <Screen footer={false}>
+      <Dialog
+        isOpen={isEffectsOpen}
+        onOpenChange={setEffectsOpen}
+        role="dialog"
+        type="flex"
+        size="large"
+      >
+        <Heading
+          slot="title"
+          level={1}
+          className={css({
+            textStyle: 'h1',
+            marginBottom: '1.5rem',
+          })}
+        >
+          {t('effects.title')}
+        </Heading>
+        <EffectsConfiguration
+          videoTrack={videoTrack}
+          onSubmit={(processor) => {
+            setProcessor(processor)
+          }}
+        />
+      </Dialog>
       <div
         className={css({
           display: 'flex',
@@ -204,8 +257,35 @@ export const Join = ({
                   </p>
                 </div>
               )}
+              <div
+                className={css({
+                  position: 'absolute',
+                  left: 0,
+                  right: 0,
+                  bottom: 0,
+                  height: '20%',
+                  backgroundImage:
+                    'linear-gradient(0deg, rgba(0,0,0,0.8) 0%, rgba(255,255,255,0) 100%)',
+                })}
+              ></div>
+              <div
+                className={css({
+                  position: 'absolute',
+                  right: 0,
+                  bottom: '0',
+                  padding: '1rem',
+                })}
+              >
+                <Button
+                  variant="whiteCircle"
+                  onPress={openEffects}
+                  tooltip={t('effects.description')}
+                  aria-label={t('effects.description')}
+                >
+                  <RiImageCircleAiFill size={24} />
+                </Button>
+              </div>
             </div>
-
             <HStack justify="center" padding={1.5}>
               <SelectToggleDevice
                 source={Track.Source.Microphone}

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts

@@ -11,7 +11,11 @@ import {
   TIMEOUT_TICK,
   timerWorkerScript,
 } from './TimerWorker'
-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+import {
+  BackgroundBlurProcessorInterface,
+  BackgroundOptions,
+  ProcessorType,
+} from '.'
 
 const PROCESSING_WIDTH = 256
 const PROCESSING_HEIGHT = 144
@@ -282,4 +286,11 @@ export class BackgroundBlurCustomProcessor
   clone() {
     return new BackgroundBlurCustomProcessor(this.options)
   }
+
+  serialize() {
+    return {
+      type: ProcessorType.BLUR,
+      options: this.options,
+    }
+  }
 }

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts

@@ -4,7 +4,11 @@ import {
   ProcessorWrapper,
 } from '@livekit/track-processors'
 import { ProcessorOptions, Track } from 'livekit-client'
-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+import {
+  BackgroundBlurProcessorInterface,
+  BackgroundOptions,
+  ProcessorType,
+} from '.'
 
 /**
  * This is simply a wrapper around track-processor-js Processor
@@ -54,4 +58,11 @@ export class BackgroundBlurTrackProcessorJsWrapper
       blurRadius: this.options!.blurRadius,
     })
   }
+
+  serialize() {
+    return {
+      type: ProcessorType.BLUR,
+      options: this.options,
+    }
+  }
 }

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -7,11 +7,21 @@ export type BackgroundOptions = {
   blurRadius?: number
 }
 
+export interface ProcessorSerialized {
+  type: ProcessorType
+  options: BackgroundOptions
+}
+
 export interface BackgroundBlurProcessorInterface
   extends TrackProcessor<Track.Kind> {
   update(opts: BackgroundOptions): void
   options: BackgroundOptions
   clone(): BackgroundBlurProcessorInterface
+  serialize(): ProcessorSerialized
+}
+
+export enum ProcessorType {
+  BLUR = 'blur',
 }
 
 export class BackgroundBlurFactory {
@@ -21,13 +31,22 @@ export class BackgroundBlurFactory {
     )
   }
 
-  static getProcessor(opts: BackgroundOptions) {
+  static getProcessor(
+    opts: BackgroundOptions
+  ): BackgroundBlurProcessorInterface | undefined {
     if (ProcessorWrapper.isSupported) {
       return new BackgroundBlurTrackProcessorJsWrapper(opts)
     }
     if (BackgroundBlurCustomProcessor.isSupported) {
       return new BackgroundBlurCustomProcessor(opts)
     }
-    return null
+    return undefined
+  }
+
+  static deserializeProcessor(data?: ProcessorSerialized) {
+    if (data?.type === ProcessorType.BLUR) {
+      return BackgroundBlurFactory.getProcessor(data?.options)
+    }
+    return undefined
   }
 }

src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx

@@ -13,7 +13,12 @@ import {
   RiVideoOffLine,
   RiVideoOnLine,
 } from '@remixicon/react'
-import { LocalAudioTrack, LocalVideoTrack, Track } from 'livekit-client'
+import {
+  LocalAudioTrack,
+  LocalVideoTrack,
+  Track,
+  VideoCaptureOptions,
+} from 'livekit-client'
 
 import { Shortcut } from '@/features/shortcuts/types'
 
@@ -21,6 +26,8 @@ import { ToggleDevice } from '@/features/rooms/livekit/components/controls/Toggl
 import { css } from '@/styled-system/css'
 import { ButtonRecipeProps } from '@/primitives/buttonRecipe'
 import { useEffect } from 'react'
+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
+import { BackgroundBlurFactory } from '../blur'
 
 export type ToggleSource = Exclude<
   Track.Source,
@@ -92,6 +99,39 @@ export const SelectToggleDevice = <T extends ToggleSource>({
   const { t } = useTranslation('rooms', { keyPrefix: 'join' })
   const trackProps = useTrackToggle(props)
 
+  const { userChoices } = usePersistentUserChoices({})
+
+  const toggle = () => {
+    if (props.source === Track.Source.Camera) {
+      /**
+       * We need to make sure that we apply the in-memory processor when re-enabling the camera.
+       * Before, we had the following bug:
+       * 1 - Configure a processor on join screen
+       * 2 - Turn off camera on join screen
+       * 3 - Join the room
+       * 4 - Turn on the camera
+       * 5 - No processor is applied to the camera
+       * Expected: The processor is applied.
+       *
+       * See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121
+       */
+      const processor = BackgroundBlurFactory.deserializeProcessor(
+        userChoices.processorSerialized
+      )
+
+      const toggle = trackProps.toggle as (
+        forceState: boolean,
+        captureOptions: VideoCaptureOptions
+      ) => Promise<void>
+
+      toggle(!trackProps.enabled, {
+        processor: processor,
+      } as VideoCaptureOptions)
+    } else {
+      trackProps.toggle()
+    }
+  }
+
   const { devices, activeDeviceId, setActiveMediaDevice } =
     useMediaDeviceSelect({ kind: config.kind, track })
 
@@ -120,6 +160,7 @@ export const SelectToggleDevice = <T extends ToggleSource>({
         {...trackProps}
         config={config}
         variant={variant}
+        toggle={toggle}
         toggleButtonProps={{
           ...(hideMenu
             ? {

src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx

@@ -2,17 +2,26 @@ import { useLocalParticipant } from '@livekit/components-react'
 import { LocalVideoTrack } from 'livekit-client'
 import { css } from '@/styled-system/css'
 import { EffectsConfiguration } from './EffectsConfiguration'
+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
 
 export const Effects = () => {
   const { cameraTrack } = useLocalParticipant()
   const localCameraTrack = cameraTrack?.track as LocalVideoTrack
+  const { saveProcessorSerialized } = usePersistentUserChoices()
+
   return (
     <div
       className={css({
         padding: '0 1.5rem',
       })}
     >
-      <EffectsConfiguration videoTrack={localCameraTrack} layout="vertical" />
+      <EffectsConfiguration
+        videoTrack={localCameraTrack}
+        layout="vertical"
+        onSubmit={(processor) =>
+          saveProcessorSerialized(processor?.serialize())
+        }
+      />
     </div>
   )
 }

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { LocalVideoTrack, Track, TrackProcessor } from 'livekit-client'
+import { LocalVideoTrack } from 'livekit-client'
 import { useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import {
@@ -33,7 +33,7 @@ export const EffectsConfiguration = ({
   layout = 'horizontal',
 }: {
   videoTrack: LocalVideoTrack
-  onSubmit?: (processor?: TrackProcessor<Track.Kind.Video>) => void
+  onSubmit?: (processor?: BackgroundBlurProcessorInterface) => void
   layout?: 'vertical' | 'horizontal'
 }) => {
   const videoRef = useRef<HTMLVideoElement>(null)
@@ -68,6 +68,8 @@ export const EffectsConfiguration = ({
         onSubmit?.(newProcessor)
       } else {
         processor?.update({ blurRadius })
+        // We want to trigger onSubmit when options changes so the parent component is aware of it.
+        onSubmit?.(processor)
       }
     } catch (error) {
       console.error('Error applying blur:', error)

src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts

@@ -0,0 +1,56 @@
+import { UsePersistentUserChoicesOptions } from '@livekit/components-react'
+import React from 'react'
+import { LocalUserChoices } from '../../routes/Room'
+import { saveUserChoices, loadUserChoices } from '@livekit/components-core'
+import { ProcessorSerialized } from '../components/blur'
+
+/**
+ * From @livekit/component-react
+ *
+ * A hook that provides access to user choices stored in local storage, such as
+ * selected media devices and their current state (on or off), as well as the user name.
+ * @alpha
+ */
+export function usePersistentUserChoices(
+  options: UsePersistentUserChoicesOptions = {}
+) {
+  const [userChoices, setSettings] = React.useState<LocalUserChoices>(
+    loadUserChoices(options.defaults, options.preventLoad ?? false)
+  )
+
+  const saveAudioInputEnabled = React.useCallback((isEnabled: boolean) => {
+    setSettings((prev) => ({ ...prev, audioEnabled: isEnabled }))
+  }, [])
+  const saveVideoInputEnabled = React.useCallback((isEnabled: boolean) => {
+    setSettings((prev) => ({ ...prev, videoEnabled: isEnabled }))
+  }, [])
+  const saveAudioInputDeviceId = React.useCallback((deviceId: string) => {
+    setSettings((prev) => ({ ...prev, audioDeviceId: deviceId }))
+  }, [])
+  const saveVideoInputDeviceId = React.useCallback((deviceId: string) => {
+    setSettings((prev) => ({ ...prev, videoDeviceId: deviceId }))
+  }, [])
+  const saveUsername = React.useCallback((username: string) => {
+    setSettings((prev) => ({ ...prev, username: username }))
+  }, [])
+  const saveProcessorSerialized = React.useCallback(
+    (processorSerialized?: ProcessorSerialized) => {
+      setSettings((prev) => ({ ...prev, processorSerialized }))
+    },
+    []
+  )
+
+  React.useEffect(() => {
+    saveUserChoices(userChoices, options.preventSave ?? false)
+  }, [userChoices, options.preventSave])
+
+  return {
+    userChoices,
+    saveAudioInputEnabled,
+    saveVideoInputEnabled,
+    saveAudioInputDeviceId,
+    saveVideoInputDeviceId,
+    saveUsername,
+    saveProcessorSerialized,
+  }
+}

src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx

@@ -1,11 +1,11 @@
 import { Track } from 'livekit-client'
 import * as React from 'react'
-import { usePersistentUserChoices } from '@livekit/components-react'
 
 import { MobileControlBar } from './MobileControlBar'
 import { DesktopControlBar } from './DesktopControlBar'
 import { SettingsDialogProvider } from '../../components/controls/SettingsDialogContext'
 import { useIsMobile } from '@/utils/useIsMobile'
+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
 
 /** @public */
 export type ControlBarControls = {

src/frontend/src/features/rooms/routes/Room.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useState } from 'react'
 import {
   usePersistentUserChoices,
-  type LocalUserChoices,
+  type LocalUserChoices as LocalUserChoicesLK,
 } from '@livekit/components-react'
 import { useParams } from 'wouter'
 import { ErrorScreen } from '@/components/ErrorScreen'
@@ -9,6 +9,11 @@ import { useUser, UserAware } from '@/features/auth'
 import { Conference } from '../components/Conference'
 import { Join } from '../components/Join'
 import { useKeyboardShortcuts } from '@/features/shortcuts/useKeyboardShortcuts'
+import { ProcessorSerialized } from '../livekit/components/blur'
+
+export type LocalUserChoices = LocalUserChoicesLK & {
+  processorSerialized?: ProcessorSerialized
+}
 
 export const Room = () => {
   const { isLoggedIn } = useUser()

src/frontend/src/features/settings/components/tabs/AccountTab.tsx

@@ -1,15 +1,13 @@
 import { A, Badge, Button, DialogProps, Field, H, P } from '@/primitives'
 import { Trans, useTranslation } from 'react-i18next'
-import {
-  usePersistentUserChoices,
-  useRoomContext,
-} from '@livekit/components-react'
+import { useRoomContext } from '@livekit/components-react'
 import { logoutUrl, useUser } from '@/features/auth'
 import { css } from '@/styled-system/css'
 import { TabPanel, TabPanelProps } from '@/primitives/Tabs'
 import { HStack } from '@/styled-system/jsx'
 import { useState } from 'react'
 import { ProConnectButton } from '@/components/ProConnectButton'
+import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices'
 
 export type AccountTabProps = Pick<DialogProps, 'onOpenChange'> &
   Pick<TabPanelProps, 'id'>

src/frontend/src/locales/en/rooms.json

@@ -18,6 +18,10 @@
       "enable": "Enable microphone",
       "label": "Microphone"
     },
+    "effects": {
+      "description": "Apply effects",
+      "title": "Effects"
+    },
     "heading": "Join the meeting",
     "joinLabel": "Join",
     "joinMeeting": "Join meeting",

src/frontend/src/locales/fr/rooms.json

@@ -19,6 +19,10 @@
       "label": "Microphone"
     },
     "heading": "Rejoindre la réunion",
+    "effects": {
+      "description": "Appliquer des effets",
+      "title": "Effets"
+    },
     "joinLabel": "Rejoindre",
     "joinMeeting": "Rejoindre la réjoindre",
     "toggleOff": "Cliquez pour désactiver",