From e71bc093bd63faac3eb01185d39adc06a9687b54 Mon Sep 17 00:00:00 2001
From: lebaudantoine <lebaud.antoine131@gmail.com>
Date: Mon, 9 Jun 2025 17:53:03 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(frontend)=20add=20RNNoise=20processor?=
 =?UTF-8?q?=20for=20meeting=20noise=20suppression?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Implement noise reduction copying Jitsi's approach. RNNoise isn't optimal
but chosen for first draft. Needs production battle-testing for CPU/RAM.

Use global audio context with pause/resume instead of deletion to avoid
WASM resource leak issues in @timephy/rnnoise-wasm dependency. Audio context
deletion may not properly release WASM resources.

Requires discussion with senior devs on resource management approach.
---
 .../livekit/processors/RnnNoiseProcessor.ts   | 102 ++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 src/frontend/src/features/rooms/livekit/processors/RnnNoiseProcessor.ts

diff --git a/src/frontend/src/features/rooms/livekit/processors/RnnNoiseProcessor.ts b/src/frontend/src/features/rooms/livekit/processors/RnnNoiseProcessor.ts
new file mode 100644
index 00000000..64b39dbe
--- /dev/null
+++ b/src/frontend/src/features/rooms/livekit/processors/RnnNoiseProcessor.ts
@@ -0,0 +1,102 @@
+import { Track, TrackProcessor, ProcessorOptions } from 'livekit-client'
+import { NoiseSuppressorWorklet_Name } from '@timephy/rnnoise-wasm'
+
+// This is an example how to get the script path using Vite, may be different when using other build tools
+// NOTE: `?worker&url` is important (`worker` to generate a working script, `url` to get its url to load it)
+import NoiseSuppressorWorklet from '@timephy/rnnoise-wasm/NoiseSuppressorWorklet?worker&url'
+
+// Use Jitsi's approach: maintain a global AudioContext variable
+// and suspend/resume it as needed to manage audio state
+let audioContext: AudioContext
+
+export interface AudioProcessorInterface
+  extends TrackProcessor<Track.Kind.Audio> {
+  name: string
+}
+
+export class RnnNoiseProcessor implements AudioProcessorInterface {
+  name: string = 'noise-reduction'
+  processedTrack?: MediaStreamTrack
+
+  private source?: MediaStreamTrack
+  private sourceNode?: MediaStreamAudioSourceNode
+  private destinationNode?: MediaStreamAudioDestinationNode
+  private noiseSuppressionNode?: AudioWorkletNode
+
+  constructor() {}
+
+  async init(opts: ProcessorOptions<Track.Kind.Audio>) {
+    if (!opts.track) {
+      throw new Error('Track is required for audio processing')
+    }
+
+    this.source = opts.track as MediaStreamTrack
+
+    if (!audioContext) {
+      audioContext = new AudioContext()
+    } else {
+      await audioContext.resume()
+    }
+
+    await audioContext.audioWorklet.addModule(NoiseSuppressorWorklet)
+
+    this.sourceNode = audioContext.createMediaStreamSource(
+      new MediaStream([this.source])
+    )
+
+    this.noiseSuppressionNode = new AudioWorkletNode(
+      audioContext,
+      NoiseSuppressorWorklet_Name
+    )
+
+    this.destinationNode = audioContext.createMediaStreamDestination()
+
+    // Connect the audio processing chain
+    this.sourceNode
+      .connect(this.noiseSuppressionNode)
+      .connect(this.destinationNode)
+
+    // Get the processed track
+    const tracks = this.destinationNode.stream.getAudioTracks()
+    if (tracks.length === 0) {
+      throw new Error('No audio tracks found for processing')
+    }
+
+    this.processedTrack = tracks[0]
+  }
+
+  async restart(opts: ProcessorOptions<Track.Kind.Audio>) {
+    await this.destroy()
+    return this.init(opts)
+  }
+
+  async destroy() {
+    // Clean up audio nodes and context
+    this.sourceNode?.disconnect()
+    this.noiseSuppressionNode?.disconnect()
+    this.destinationNode?.disconnect()
+
+    /**
+     * Audio Context Lifecycle Management
+     *
+     * We prefer suspending the audio context rather than destroying and recreating it
+     * to avoid memory leaks in WebAssembly-based audio processing.
+     *
+     * Issue: When an AudioContext containing WebAssembly modules is destroyed,
+     * the WASM resources are not properly garbage collected. This causes:
+     * - Retained JavaScript VM instances
+     * - Growing memory consumption over multiple create/destroy cycles
+     * - Potential performance degradation
+     *
+     * Solution: Use suspend() and resume() methods instead of close() to maintain
+     * the same context instance while controlling audio processing state.
+     */
+    await audioContext.suspend()
+
+    this.sourceNode = undefined
+    this.destinationNode = undefined
+    this.source = undefined
+    this.processedTrack = undefined
+    this.noiseSuppressionNode = undefined
+  }
+}