From 897552c8c52ebc5ad6b44d0ab943e689787cd6af Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 00:54:07 +0200 Subject: [PATCH 01/28] feat(audio): add RNNoise processor and publisher wiring Signed-off-by: LucaPisl --- package.json | 5 +- src/audio/RNNoiseProcessor.ts | 287 ++++++++++++++++++ src/settings/settings.ts | 4 + src/state/CallViewModel/CallViewModel.ts | 4 +- .../CallViewModel/localMember/Publisher.ts | 90 ++++++ yarn.lock | 8 + 6 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 src/audio/RNNoiseProcessor.ts diff --git a/package.json b/package.json index 705b0f103..d532c0e74 100644 --- a/package.json +++ b/package.json @@ -140,5 +140,8 @@ "@livekit/components-core/rxjs": "^7.8.1", "@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18" }, - "packageManager": "yarn@4.7.0" + "packageManager": "yarn@4.7.0", + "dependencies": { + "@jitsi/rnnoise-wasm": "^0.2.1" + } } diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts new file mode 100644 index 000000000..b7a5fc6b0 --- /dev/null +++ b/src/audio/RNNoiseProcessor.ts @@ -0,0 +1,287 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import type { + AudioProcessorOptions, + Track, + TrackProcessor, +} from "livekit-client"; + +/** + * The number of samples per frame expected by RNNoise (at 48kHz = 10ms). + */ +const RNNOISE_SAMPLE_LENGTH = 480; +const RNNOISE_WORKLET_NAME = "rnnoise-processor"; +const loadedAudioWorklets = new WeakSet(); + +/** + * Whether the current runtime supports the required APIs for RNNoise. + */ +export function supportsRNNoiseProcessor(): boolean { + return ( + typeof AudioWorkletNode !== "undefined" && + typeof MediaStreamAudioDestinationNode !== "undefined" && + typeof MediaStreamAudioSourceNode !== "undefined" + ); +} + +/** + * Generates the AudioWorklet processor code as a string. + * + * The worklet loads the RNNoise WASM module synchronously (base64-inlined) + * and processes audio in 480-sample frames. A ring buffer bridges the + * 128-sample AudioWorklet blocks to the 480-sample RNNoise frames. + */ +function createWorkletCode(rnnoiseModuleCode: string): string { + // Patch the rnnoise-sync.js for AudioWorklet scope: + // 1. Replace import.meta.url — not available in classic worklet scripts + // 2. Remove the ES module export statement + const patched = rnnoiseModuleCode + .replace(/import\.meta\.url/g, '""') + .replace(/export\s+default\s+createRNNWasmModuleSync;?\s*$/m, ""); + + return ` +${patched} + +const FRAME_SIZE = ${RNNOISE_SAMPLE_LENGTH}; +const RING_SIZE = FRAME_SIZE * 3; // Enough headroom for buffering + +class RNNoiseWorkletProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._ready = false; + this._destroyed = false; + + // Ring buffers + this._inBuf = new Float32Array(RING_SIZE); + this._outBuf = new Float32Array(RING_SIZE); + this._inW = 0; // input write position + this._inR = 0; // input read position + this._outW = 0; // output write position + this._outR = 0; // output read position + + this._initRNNoise(); + + this.port.onmessage = (event) => { + if (event.data.type === 'destroy') { + this._cleanup(); + } + }; + } + + _ringAvailable(w, r) { + let avail = w - r; + if (avail < 0) avail += RING_SIZE; + return avail; + } + + _initRNNoise() { + try { + const module = createRNNWasmModuleSync(); + + // Allocate a buffer in WASM memory for one frame of float32 samples + const pcmBuf = module._malloc(FRAME_SIZE * 4); + module._rnnoise_init(); + const state = module._rnnoise_create(); + + this._module = module; + this._pcmBuf = pcmBuf; + this._state = state; + this._heapF32 = module.HEAPF32; + + this._ready = true; + } catch (e) { + // If RNNoise fails to initialize, audio will pass through unprocessed + this.port.postMessage({ type: 'error', message: String(e) }); + } + } + + _cleanup() { + if (this._module && this._state) { + this._module._rnnoise_destroy(this._state); + this._module._free(this._pcmBuf); + this._state = null; + } + this._destroyed = true; + } + + _processRNNoiseFrame() { + const heapIdx = this._pcmBuf >> 2; // byte offset → float32 index + + // Copy from input ring buffer to WASM heap, scaling to int16 range + for (let i = 0; i < FRAME_SIZE; i++) { + this._heapF32[heapIdx + i] = + this._inBuf[(this._inR + i) % RING_SIZE] * 32768.0; + } + this._inR = (this._inR + FRAME_SIZE) % RING_SIZE; + + // Run RNNoise denoising (in-place) + this._module._rnnoise_process_frame( + this._state, this._pcmBuf, this._pcmBuf + ); + + // Copy from WASM heap to output ring buffer, scaling back to float range + for (let i = 0; i < FRAME_SIZE; i++) { + this._outBuf[(this._outW + i) % RING_SIZE] = + this._heapF32[heapIdx + i] / 32768.0; + } + this._outW = (this._outW + FRAME_SIZE) % RING_SIZE; + } + + process(inputs, outputs) { + if (this._destroyed) return false; + + const input = inputs[0]?.[0]; + const output = outputs[0]?.[0]; + + if (!input || !output) return true; + + if (!this._ready) { + // Pass through until RNNoise is ready + output.set(input); + return true; + } + + const blockSize = input.length; + + // Write input samples to the input ring buffer + for (let i = 0; i < blockSize; i++) { + this._inBuf[this._inW] = input[i]; + this._inW = (this._inW + 1) % RING_SIZE; + } + + // Process complete frames + while (this._ringAvailable(this._inW, this._inR) >= FRAME_SIZE) { + this._processRNNoiseFrame(); + } + + // Read from output ring buffer + const outAvail = this._ringAvailable(this._outW, this._outR); + const toRead = Math.min(blockSize, outAvail); + + for (let i = 0; i < toRead; i++) { + output[i] = this._outBuf[this._outR]; + this._outR = (this._outR + 1) % RING_SIZE; + } + // Fill remaining with silence (only during initial buffering) + for (let i = toRead; i < blockSize; i++) { + output[i] = 0; + } + + return true; + } +} + +registerProcessor('${RNNOISE_WORKLET_NAME}', RNNoiseWorkletProcessor); +`; +} + +/** + * A LiveKit TrackProcessor that applies RNNoise-based noise suppression + * to a local audio track via an AudioWorklet. + * + * The RNNoise WASM binary is lazy-loaded only when the processor is + * initialized, keeping the main bundle small. + */ +export class RNNoiseProcessor implements TrackProcessor< + Track.Kind.Audio, + AudioProcessorOptions +> { + public name = "rnnoise-noise-suppression"; + public processedTrack?: MediaStreamTrack; + + private sourceNode?: MediaStreamAudioSourceNode; + private workletNode?: AudioWorkletNode; + private destinationNode?: MediaStreamAudioDestinationNode; + private blobUrl?: string; + private destroyed = false; + + private async ensureWorkletRegistered( + audioContext: AudioContext, + ): Promise { + if (loadedAudioWorklets.has(audioContext)) { + return; + } + + // Lazy-load the RNNoise sync WASM module code + const rnnoiseModule = + await import("@jitsi/rnnoise-wasm/dist/rnnoise-sync.js?raw"); + const workletCode = createWorkletCode( + rnnoiseModule.default as unknown as string, + ); + + // Create a Blob URL for the AudioWorklet module. + const blob = new Blob([workletCode], { type: "text/javascript" }); + this.blobUrl = URL.createObjectURL(blob); + + await audioContext.audioWorklet.addModule(this.blobUrl); + loadedAudioWorklets.add(audioContext); + } + + public async init(opts: AudioProcessorOptions): Promise { + this.destroyed = false; + const { audioContext, track } = opts; + + await this.ensureWorkletRegistered(audioContext); + + // Build the audio processing graph: + // MediaStreamSource → AudioWorkletNode (RNNoise) → MediaStreamDestination + const sourceNode = audioContext.createMediaStreamSource( + new MediaStream([track]), + ); + const workletNode = new AudioWorkletNode( + audioContext, + RNNOISE_WORKLET_NAME, + { + channelCount: 1, + channelCountMode: "explicit", + }, + ); + const destinationNode = audioContext.createMediaStreamDestination(); + + sourceNode.connect(workletNode); + workletNode.connect(destinationNode); + + this.sourceNode = sourceNode; + this.workletNode = workletNode; + this.destinationNode = destinationNode; + this.processedTrack = destinationNode.stream.getAudioTracks()[0]; + } + + public async restart(opts: AudioProcessorOptions): Promise { + await this.destroy(); + await this.init(opts); + } + + public async destroy(): Promise { + if (this.destroyed) { + await Promise.resolve(); + return; + } + this.destroyed = true; + + // Signal the worklet to clean up WASM resources + this.workletNode?.port.postMessage({ type: "destroy" }); + + // Disconnect the audio graph + this.sourceNode?.disconnect(); + this.workletNode?.disconnect(); + this.destinationNode?.disconnect(); + + // Revoke the Blob URL + if (this.blobUrl) { + URL.revokeObjectURL(this.blobUrl); + this.blobUrl = undefined; + } + + this.sourceNode = undefined; + this.workletNode = undefined; + this.destinationNode = undefined; + this.processedTrack = undefined; + await Promise.resolve(); + } +} diff --git a/src/settings/settings.ts b/src/settings/settings.ts index a674f1aae..8cc76803b 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -97,6 +97,10 @@ export const videoInput = new Setting( ); export const backgroundBlur = new Setting("background-blur", false); +export const rnnoiseNoiseSuppression = new Setting( + "rnnoise-noise-suppression", + false, +); export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index c19c4818d..b95550fb4 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -61,6 +61,7 @@ import { duplicateTiles, MatrixRTCMode, playReactionsSound, + rnnoiseNoiseSuppression, showReactions, } from "../../settings/settings"; import { isFirefox } from "../../Platform"; @@ -480,7 +481,8 @@ export function createCallViewModel$( getUrlParams().controlledAudioDevices, options.livekitRoomFactory, getUrlParams().echoCancellation, - getUrlParams().noiseSuppression, + (getUrlParams().noiseSuppression ?? true) && + !rnnoiseNoiseSuppression.getValue(), ); const connectionManager = createConnectionManager$({ diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index b7841c498..50e27b157 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. */ import { ConnectionState as LivekitConnectionState, + type LocalAudioTrack, type LocalTrackPublication, LocalVideoTrack, ParticipantEvent, @@ -14,9 +15,12 @@ import { Track, } from "livekit-client"; import { + combineLatest, + distinctUntilChanged, map, NEVER, type Observable, + skip, type Subscription, switchMap, } from "rxjs"; @@ -33,6 +37,11 @@ import { getUrlParams } from "../../../UrlParams.ts"; import { observeTrackReference$ } from "../../observeTrackReference"; import { type Connection } from "../remoteMembers/Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; +import { + RNNoiseProcessor, + supportsRNNoiseProcessor, +} from "../../../audio/RNNoiseProcessor.ts"; +import { rnnoiseNoiseSuppression } from "../../../settings/settings.ts"; /** * A wrapper for a Connection object. @@ -73,6 +82,8 @@ export class Publisher { // Setup track processor syncing (blur) this.observeTrackProcessors(this.scope, room, trackerProcessorState$); + this.observeRNNoiseProcessor(this.scope, room); + this.observeRNNoiseSettingRestart(this.scope, room, devices); // Observe media device changes and update LiveKit active devices accordingly this.observeMediaDevices(this.scope, devices, controlledAudioDevices); @@ -416,4 +427,83 @@ export class Publisher { ); trackProcessorSync(scope, track$, trackerProcessorState$); } + + private observeRNNoiseProcessor( + scope: ObservableScope, + room: LivekitRoom, + ): void { + const microphoneTrack$ = scope.behavior( + observeTrackReference$( + room.localParticipant, + Track.Source.Microphone, + ).pipe( + map((trackRef) => { + const track = trackRef?.publication.track; + return track?.kind === Track.Kind.Audio + ? (track as LocalAudioTrack) + : null; + }), + ), + null, + ); + + combineLatest([microphoneTrack$, rnnoiseNoiseSuppression.value$]) + .pipe( + scope.bind(), + distinctUntilChanged(([aTrack, aEnabled], [bTrack, bEnabled]) => { + return aTrack === bTrack && aEnabled === bEnabled; + }), + ) + .subscribe(([microphoneTrack, rnnoiseEnabled]) => { + if (!microphoneTrack || !supportsRNNoiseProcessor()) return; + + void this.syncRNNoiseProcessor(microphoneTrack, rnnoiseEnabled); + }); + } + + private observeRNNoiseSettingRestart( + scope: ObservableScope, + room: LivekitRoom, + devices: MediaDevices, + ): void { + rnnoiseNoiseSuppression.value$ + .pipe(scope.bind(), distinctUntilChanged(), skip(1)) + .subscribe((rnnoiseEnabled) => { + const audioTrack = room.localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.audioTrack; + if (!audioTrack) return; + + const { echoCancellation = true, noiseSuppression = true } = + getUrlParams(); + + void audioTrack + .restartTrack({ + deviceId: devices.audioInput.selected$.value?.id, + echoCancellation, + noiseSuppression: noiseSuppression && !rnnoiseEnabled, + }) + .catch((e) => { + this.logger.error("Failed to restart microphone track", e); + }); + }); + } + + private async syncRNNoiseProcessor( + microphoneTrack: LocalAudioTrack, + rnnoiseEnabled: boolean, + ): Promise { + try { + const processor = microphoneTrack.getProcessor(); + const rnnoiseActive = processor?.name === "rnnoise-noise-suppression"; + + if (rnnoiseEnabled && !rnnoiseActive) { + await microphoneTrack.setProcessor(new RNNoiseProcessor()); + } else if (!rnnoiseEnabled && rnnoiseActive) { + await microphoneTrack.stopProcessor(); + } + } catch (e) { + this.logger.error("Failed to apply RNNoise microphone processor", e); + } + } } diff --git a/yarn.lock b/yarn.lock index 4675d0e1a..dccfb8ebf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3195,6 +3195,13 @@ __metadata: languageName: node linkType: hard +"@jitsi/rnnoise-wasm@npm:^0.2.1": + version: 0.2.1 + resolution: "@jitsi/rnnoise-wasm@npm:0.2.1" + checksum: 10c0/6e5b475b364660eb24c0fa9843a63040253c2ce4034de9313e811448f5c6dad2205a0f22d3a9ef15cbef3c808941b0681d238d53d5a853e194a4c88cdd5569b1 + languageName: node + linkType: hard + "@jridgewell/gen-mapping@npm:^0.3.12": version: 0.3.12 resolution: "@jridgewell/gen-mapping@npm:0.3.12" @@ -8377,6 +8384,7 @@ __metadata: "@fontsource/inter": "npm:^5.1.0" "@formatjs/intl-durationformat": "npm:^0.10.0" "@formatjs/intl-segmenter": "npm:^11.7.3" + "@jitsi/rnnoise-wasm": "npm:^0.2.1" "@livekit/components-core": "npm:^0.12.0" "@livekit/components-react": "npm:^2.0.0" "@livekit/protocol": "npm:^1.42.2" From 222bda898441f998a6ab93a5adcdcc5e12c1c660 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 00:54:13 +0200 Subject: [PATCH 02/28] feat(settings): add RNNoise audio toggle and translations Signed-off-by: LucaPisl --- locales/en/app.json | 5 ++++- src/settings/SettingsModal.tsx | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/locales/en/app.json b/locales/en/app.json index 0b0ac7b4d..79a6de1c9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -196,7 +196,10 @@ "settings": { "audio_tab": { "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.", - "effect_volume_label": "Sound effect volume" + "effect_volume_label": "Sound effect volume", + "rnnoise_header": "Noise suppression", + "rnnoise_label": "Enable enhanced noise suppression (RNNoise)", + "rnnoise_not_supported": "(Enhanced noise suppression is not supported by this browser.)" }, "background_blur_header": "Background", "background_blur_label": "Blur the background of the video", diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 2b4078aa5..a1ea920b6 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -24,6 +24,7 @@ import { soundEffectVolume as soundEffectVolumeSetting, backgroundBlur as backgroundBlurSetting, developerMode, + rnnoiseNoiseSuppression as rnnoiseNoiseSuppressionSetting, } from "./settings"; import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; @@ -34,6 +35,7 @@ import { FieldRow, InputField } from "../input/Input"; import { useSubmitRageshake } from "./submit-rageshake"; import { useUrlParams } from "../UrlParams"; import { useBehavior } from "../useBehavior"; +import { supportsRNNoiseProcessor } from "../audio/RNNoiseProcessor"; type SettingsTab = | "audio" @@ -98,6 +100,32 @@ export const SettingsModal: FC = ({ ); }; + const RNNoiseCheckbox: React.FC = (): ReactNode => { + const supported = supportsRNNoiseProcessor(); + const [rnnoiseEnabled, setRnnoiseEnabled] = useSetting( + rnnoiseNoiseSuppressionSetting, + ); + + return ( + <> +

{t("settings.audio_tab.rnnoise_header")}

+ + setRnnoiseEnabled(e.target.checked)} + disabled={!supported} + /> + + + ); + }; + const devices = useMediaDevices(); useEffect(() => { if (open) devices.requestDeviceNames(); @@ -164,6 +192,8 @@ export const SettingsModal: FC = ({ step={0.01} /> + + ), From 60f2396b22bbe17f6d3151b7d6e879467ddfd18e Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 00:54:19 +0200 Subject: [PATCH 03/28] test(audio): add RNNoise processor and publisher coverage Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 197 ++++++++++++++++++ .../localMember/Publisher.test.ts | 105 ++++++++++ 2 files changed, 302 insertions(+) create mode 100644 src/audio/RNNoiseProcessor.test.ts diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts new file mode 100644 index 000000000..2ccb2d103 --- /dev/null +++ b/src/audio/RNNoiseProcessor.test.ts @@ -0,0 +1,197 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Track } from "livekit-client"; + +import { RNNoiseProcessor, supportsRNNoiseProcessor } from "./RNNoiseProcessor"; + +vi.mock("@jitsi/rnnoise-wasm/dist/rnnoise-sync.js?raw", () => ({ + default: + "function createRNNWasmModuleSync(){}; export default createRNNWasmModuleSync;", +})); + +type TestContext = { + addModule: ReturnType; + createSourceNode: ReturnType; + createDestinationNode: ReturnType; + sourceNode: MediaStreamAudioSourceNode; + destinationNode: MediaStreamAudioDestinationNode; + processedTrack: MediaStreamTrack; + workletNode: AudioWorkletNode; + audioContext: AudioContext; + track: MediaStreamTrack; +}; + +function createTestContext(): TestContext { + const processedTrack = { id: "processed-track" } as MediaStreamTrack; + const sourceNode = { + connect: vi.fn(), + disconnect: vi.fn(), + } as unknown as MediaStreamAudioSourceNode; + const destinationNode = { + stream: { + getAudioTracks: () => [processedTrack], + }, + disconnect: vi.fn(), + } as unknown as MediaStreamAudioDestinationNode; + const workletNode = { + connect: vi.fn(), + disconnect: vi.fn(), + port: { + postMessage: vi.fn(), + }, + } as unknown as AudioWorkletNode; + const addModule = vi.fn().mockResolvedValue(undefined); + const createSourceNode = vi.fn().mockReturnValue(sourceNode); + const createDestinationNode = vi.fn().mockReturnValue(destinationNode); + const audioContext = { + audioWorklet: { addModule }, + createMediaStreamSource: createSourceNode, + createMediaStreamDestination: createDestinationNode, + } as unknown as AudioContext; + const track = { + id: "input-track", + kind: Track.Kind.Audio, + } as MediaStreamTrack; + + return { + addModule, + createSourceNode, + createDestinationNode, + sourceNode, + destinationNode, + processedTrack, + workletNode, + audioContext, + track, + }; +} + +describe("RNNoiseProcessor", () => { + beforeEach(() => { + vi.stubGlobal( + "URL", + Object.assign(URL, { + createObjectURL: vi.fn().mockReturnValue("blob:rnnoise"), + revokeObjectURL: vi.fn(), + }), + ); + vi.stubGlobal( + "MediaStream", + class MediaStream { + public constructor(_tracks?: MediaStreamTrack[]) {} + }, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it("initializes audio graph and exposes processed track", async () => { + const t = createTestContext(); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + expect(t.addModule).toHaveBeenCalledWith("blob:rnnoise"); + expect(t.createSourceNode).toHaveBeenCalledOnce(); + expect(t.createDestinationNode).toHaveBeenCalledOnce(); + expect(processor.processedTrack).toBe(t.processedTrack); + }); + + it("destroys processing graph and is idempotent", async () => { + const t = createTestContext(); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + await processor.destroy(); + await processor.destroy(); + + expect(t.sourceNode.disconnect).toHaveBeenCalledOnce(); + expect(t.workletNode.disconnect).toHaveBeenCalledOnce(); + expect(t.destinationNode.disconnect).toHaveBeenCalledOnce(); + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "destroy", + }); + expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:rnnoise"); + expect(processor.processedTrack).toBeUndefined(); + }); + + it("restart re-initializes with a new processed track", async () => { + const first = createTestContext(); + const second = createTestContext(); + const workletCtor = vi + .fn() + .mockReturnValueOnce(first.workletNode) + .mockReturnValueOnce(second.workletNode); + vi.stubGlobal("AudioWorkletNode", workletCtor); + + const processor = new RNNoiseProcessor(); + await processor.init({ + kind: Track.Kind.Audio, + track: first.track, + audioContext: first.audioContext, + }); + const firstProcessedTrack = processor.processedTrack; + + await processor.restart({ + kind: Track.Kind.Audio, + track: second.track, + audioContext: second.audioContext, + }); + + expect(processor.processedTrack).toBe(second.processedTrack); + expect(processor.processedTrack).not.toBe(firstProcessedTrack); + }); + + it("loads the worklet module once per AudioContext", async () => { + const t = createTestContext(); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + const firstProcessor = new RNNoiseProcessor(); + const secondProcessor = new RNNoiseProcessor(); + + await firstProcessor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + await secondProcessor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + expect(t.addModule).toHaveBeenCalledOnce(); + }); + + it("reports support based on AudioWorklet availability", () => { + expect(supportsRNNoiseProcessor()).toBe(false); + vi.stubGlobal("AudioWorkletNode", class AudioWorkletNode {}); + vi.stubGlobal( + "MediaStreamAudioDestinationNode", + class MediaStreamAudioDestinationNode {}, + ); + vi.stubGlobal( + "MediaStreamAudioSourceNode", + class MediaStreamAudioSourceNode {}, + ); + expect(supportsRNNoiseProcessor()).toBe(true); + }); +}); diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index a0eaa2fd6..d121d3a2d 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -27,6 +27,7 @@ import { import { Publisher } from "./Publisher"; import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; +import { rnnoiseNoiseSuppression } from "../../../settings/settings"; let scope: ObservableScope; @@ -37,8 +38,12 @@ beforeEach(() => { afterEach(() => scope.end()); function createMockLocalTrack(source: Track.Source): LocalTrack { + let processor: { name: string } | undefined; + const kind = + source === Track.Source.Microphone ? Track.Kind.Audio : Track.Kind.Video; const track = { source, + kind, isMuted: false, isUpstreamPaused: false, } as Partial as LocalTrack; @@ -57,6 +62,16 @@ function createMockLocalTrack(source: Track.Source): LocalTrack { // @ts-expect-error - for that test we want to set isUpstreamPaused directly track.isUpstreamPaused = false; }); + vi.mocked(track).getProcessor = vi.fn().mockImplementation(() => processor); + vi.mocked(track).setProcessor = vi + .fn() + .mockImplementation((nextProcessor) => { + processor = nextProcessor as { name: string }; + }); + vi.mocked(track).stopProcessor = vi.fn().mockImplementation(() => { + processor = undefined; + }); + vi.mocked(track).restartTrack = vi.fn().mockResolvedValue(undefined); return track; } @@ -96,6 +111,7 @@ let trackPublications: LocalTrackPublication[]; let createTrackLock: Promise; beforeEach(() => { + rnnoiseNoiseSuppression.setValue(false); trackPublications = []; audioEnabled$ = new BehaviorSubject(false); videoEnabled$ = new BehaviorSubject(false); @@ -339,6 +355,95 @@ describe("Publisher", () => { }); it("does mute unmute audio", async () => {}); + + describe("RNNoise", () => { + beforeEach(() => { + vi.stubGlobal("AudioWorkletNode", class AudioWorkletNode {}); + vi.stubGlobal( + "MediaStreamAudioDestinationNode", + class MediaStreamAudioDestinationNode {}, + ); + vi.stubGlobal( + "MediaStreamAudioSourceNode", + class MediaStreamAudioSourceNode {}, + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + rnnoiseNoiseSuppression.setValue(false); + }); + + it("enabling setting applies RNNoise processor on microphone track", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { setProcessor: (...args: unknown[]) => void }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + }); + + it("disabling setting removes RNNoise processor on microphone track", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + setProcessor: (...args: unknown[]) => void; + stopProcessor: () => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + rnnoiseNoiseSuppression.setValue(false); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + expect(micTrack.stopProcessor).toHaveBeenCalledOnce(); + }); + + it("restarts microphone track with native noise suppression disabled when RNNoise is enabled", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { restartTrack: (...args: unknown[]) => void }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + + expect(micTrack.restartTrack).toHaveBeenCalledWith( + expect.objectContaining({ + noiseSuppression: false, + }), + ); + }); + }); }); describe("Bug fix", () => { From 9045d237ea5b955fa4155b3b9c60546534ac0790 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 01:10:44 +0200 Subject: [PATCH 04/28] feat(audio): add configurable RNNoise suppression presets Signed-off-by: LucaPisl --- locales/en/app.json | 6 +- src/audio/RNNoiseProcessor.test.ts | 25 +++- src/audio/RNNoiseProcessor.ts | 109 +++++++++++++++++- src/audio/rnnoiseTypes.ts | 15 +++ src/settings/SettingsModal.tsx | 58 +++++++++- src/settings/settings.ts | 6 + .../localMember/Publisher.test.ts | 34 +++++- .../CallViewModel/localMember/Publisher.ts | 47 ++++++-- 8 files changed, 282 insertions(+), 18 deletions(-) create mode 100644 src/audio/rnnoiseTypes.ts diff --git a/locales/en/app.json b/locales/en/app.json index 79a6de1c9..ce7e0c2f7 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -199,7 +199,11 @@ "effect_volume_label": "Sound effect volume", "rnnoise_header": "Noise suppression", "rnnoise_label": "Enable enhanced noise suppression (RNNoise)", - "rnnoise_not_supported": "(Enhanced noise suppression is not supported by this browser.)" + "rnnoise_not_supported": "(Enhanced noise suppression is not supported by this browser.)", + "rnnoise_preset_balanced": "Balanced", + "rnnoise_preset_conservative": "Conservative", + "rnnoise_preset_description": "Pick a suppression profile. Stronger modes remove more keyboard noise but can sound more processed.", + "rnnoise_preset_strong": "Strong" }, "background_blur_header": "Background", "background_blur_label": "Blur the background of the video", diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index 2ccb2d103..dcdaf2675 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -97,7 +97,7 @@ describe("RNNoiseProcessor", () => { it("initializes audio graph and exposes processed track", async () => { const t = createTestContext(); vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); - const processor = new RNNoiseProcessor(); + const processor = new RNNoiseProcessor("balanced"); await processor.init({ kind: Track.Kind.Audio, @@ -108,6 +108,10 @@ describe("RNNoiseProcessor", () => { expect(t.addModule).toHaveBeenCalledWith("blob:rnnoise"); expect(t.createSourceNode).toHaveBeenCalledOnce(); expect(t.createDestinationNode).toHaveBeenCalledOnce(); + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "preset", + preset: "balanced", + }); expect(processor.processedTrack).toBe(t.processedTrack); }); @@ -194,4 +198,23 @@ describe("RNNoiseProcessor", () => { ); expect(supportsRNNoiseProcessor()).toBe(true); }); + + it("updates worklet preset at runtime", async () => { + const t = createTestContext(); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + processor.setPreset("strong"); + + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "preset", + preset: "strong", + }); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index b7a5fc6b0..b013a2ffe 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -10,12 +10,14 @@ import type { Track, TrackProcessor, } from "livekit-client"; +import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; /** * The number of samples per frame expected by RNNoise (at 48kHz = 10ms). */ const RNNOISE_SAMPLE_LENGTH = 480; const RNNOISE_WORKLET_NAME = "rnnoise-processor"; +const DEFAULT_RNNOISE_PRESET: RNNoiseSuppressionPreset = "conservative"; const loadedAudioWorklets = new WeakSet(); /** @@ -49,6 +51,34 @@ ${patched} const FRAME_SIZE = ${RNNOISE_SAMPLE_LENGTH}; const RING_SIZE = FRAME_SIZE * 3; // Enough headroom for buffering +const SAMPLE_RATE = 48000; +const DEFAULT_PRESET = "${DEFAULT_RNNOISE_PRESET}"; +const PRESETS = { + conservative: { + maxAttenuationDb: 4, + openThreshold: 0.92, + closeThreshold: 0.60, + holdFrames: 12, + attenuateMs: 120, + releaseMs: 25, + }, + balanced: { + maxAttenuationDb: 8, + openThreshold: 0.90, + closeThreshold: 0.55, + holdFrames: 10, + attenuateMs: 90, + releaseMs: 22, + }, + strong: { + maxAttenuationDb: 12, + openThreshold: 0.88, + closeThreshold: 0.50, + holdFrames: 9, + attenuateMs: 70, + releaseMs: 18, + }, +}; class RNNoiseWorkletProcessor extends AudioWorkletProcessor { constructor() { @@ -63,16 +93,66 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._inR = 0; // input read position this._outW = 0; // output write position this._outR = 0; // output read position + this._currentGain = 1; + this._targetGain = 1; + this._holdFrames = 0; + + this._setPreset(DEFAULT_PRESET); this._initRNNoise(); this.port.onmessage = (event) => { if (event.data.type === 'destroy') { this._cleanup(); + } else if (event.data.type === 'preset') { + this._setPreset(event.data.preset); } }; } + _smoothingStepFromMs(ms) { + if (ms <= 0) return 1; + const tau = ms / 1000; + return 1 - Math.exp(-1 / (SAMPLE_RATE * tau)); + } + + _setPreset(preset) { + if (!(preset in PRESETS)) return; + this._preset = preset; + const config = PRESETS[preset]; + this._maxAttenuationDb = config.maxAttenuationDb; + this._openThreshold = config.openThreshold; + this._closeThreshold = config.closeThreshold; + this._holdFramesConfig = config.holdFrames; + this._attenuateStep = this._smoothingStepFromMs(config.attenuateMs); + this._releaseStep = this._smoothingStepFromMs(config.releaseMs); + } + + _updateTargetGain(vadProbability) { + if (vadProbability >= this._openThreshold) { + this._holdFrames = this._holdFramesConfig; + this._targetGain = 1; + return; + } + + if (this._holdFrames > 0) { + this._holdFrames -= 1; + this._targetGain = 1; + return; + } + + const thresholdRange = this._openThreshold - this._closeThreshold; + const attenuationProgress = thresholdRange > 0 + ? Math.max( + 0, + Math.min(1, (this._openThreshold - vadProbability) / thresholdRange), + ) + : 1; + + const attenuationDb = attenuationProgress * this._maxAttenuationDb; + this._targetGain = Math.pow(10, -attenuationDb / 20); + } + _ringAvailable(w, r) { let avail = w - r; if (avail < 0) avail += RING_SIZE; @@ -120,14 +200,21 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._inR = (this._inR + FRAME_SIZE) % RING_SIZE; // Run RNNoise denoising (in-place) - this._module._rnnoise_process_frame( + const vadProbability = this._module._rnnoise_process_frame( this._state, this._pcmBuf, this._pcmBuf ); + this._updateTargetGain(vadProbability); - // Copy from WASM heap to output ring buffer, scaling back to float range + // Copy from WASM heap to output ring buffer, scaling back to float range. + // Apply additional conservative attenuation between speech segments. for (let i = 0; i < FRAME_SIZE; i++) { + const smoothingStep = this._targetGain < this._currentGain + ? this._attenuateStep + : this._releaseStep; + this._currentGain += + (this._targetGain - this._currentGain) * smoothingStep; this._outBuf[(this._outW + i) % RING_SIZE] = - this._heapF32[heapIdx + i] / 32768.0; + (this._heapF32[heapIdx + i] / 32768.0) * this._currentGain; } this._outW = (this._outW + FRAME_SIZE) % RING_SIZE; } @@ -199,6 +286,13 @@ export class RNNoiseProcessor implements TrackProcessor< private destinationNode?: MediaStreamAudioDestinationNode; private blobUrl?: string; private destroyed = false; + private preset: RNNoiseSuppressionPreset; + + public constructor( + preset: RNNoiseSuppressionPreset = DEFAULT_RNNOISE_PRESET, + ) { + this.preset = preset; + } private async ensureWorkletRegistered( audioContext: AudioContext, @@ -249,6 +343,7 @@ export class RNNoiseProcessor implements TrackProcessor< this.sourceNode = sourceNode; this.workletNode = workletNode; this.destinationNode = destinationNode; + this.workletNode.port.postMessage({ type: "preset", preset: this.preset }); this.processedTrack = destinationNode.stream.getAudioTracks()[0]; } @@ -284,4 +379,12 @@ export class RNNoiseProcessor implements TrackProcessor< this.processedTrack = undefined; await Promise.resolve(); } + + public setPreset(preset: RNNoiseSuppressionPreset): void { + this.preset = preset; + this.workletNode?.port.postMessage({ + type: "preset", + preset: this.preset, + }); + } } diff --git a/src/audio/rnnoiseTypes.ts b/src/audio/rnnoiseTypes.ts new file mode 100644 index 000000000..6d6d964f0 --- /dev/null +++ b/src/audio/rnnoiseTypes.ts @@ -0,0 +1,15 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +export const rnnoiseSuppressionPresets = [ + "conservative", + "balanced", + "strong", +] as const; + +export type RNNoiseSuppressionPreset = + (typeof rnnoiseSuppressionPresets)[number]; diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index a1ea920b6..58d19596f 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -5,10 +5,24 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type FC, type ReactNode, useEffect, useState } from "react"; +import { + type ChangeEvent, + type FC, + type ReactNode, + useEffect, + useId, + useState, +} from "react"; import { useTranslation } from "react-i18next"; import { type MatrixClient } from "matrix-js-sdk"; -import { Button, Root as Form, Separator } from "@vector-im/compound-web"; +import { + Button, + InlineField, + Label, + RadioControl, + Root as Form, + Separator, +} from "@vector-im/compound-web"; import { type Room as LivekitRoom } from "livekit-client"; import { Modal } from "../Modal"; @@ -25,6 +39,7 @@ import { backgroundBlur as backgroundBlurSetting, developerMode, rnnoiseNoiseSuppression as rnnoiseNoiseSuppressionSetting, + rnnoiseNoiseSuppressionPreset as rnnoiseNoiseSuppressionPresetSetting, } from "./settings"; import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; @@ -36,6 +51,10 @@ import { useSubmitRageshake } from "./submit-rageshake"; import { useUrlParams } from "../UrlParams"; import { useBehavior } from "../useBehavior"; import { supportsRNNoiseProcessor } from "../audio/RNNoiseProcessor"; +import { + type RNNoiseSuppressionPreset, + rnnoiseSuppressionPresets, +} from "../audio/rnnoiseTypes"; type SettingsTab = | "audio" @@ -105,6 +124,20 @@ export const SettingsModal: FC = ({ const [rnnoiseEnabled, setRnnoiseEnabled] = useSetting( rnnoiseNoiseSuppressionSetting, ); + const [rnnoisePreset, setRnnoisePreset] = useSetting( + rnnoiseNoiseSuppressionPresetSetting, + ); + const rnnoisePresetGroup = useId(); + + const onPresetChange = (e: ChangeEvent): void => { + setRnnoisePreset(e.target.value as RNNoiseSuppressionPreset); + }; + + const presetLabelByPreset: Record = { + conservative: t("settings.audio_tab.rnnoise_preset_conservative"), + balanced: t("settings.audio_tab.rnnoise_preset_balanced"), + strong: t("settings.audio_tab.rnnoise_preset_strong"), + }; return ( <> @@ -122,6 +155,27 @@ export const SettingsModal: FC = ({ disabled={!supported} /> + {rnnoiseEnabled && ( + <> +

{t("settings.audio_tab.rnnoise_preset_description")}

+ {rnnoiseSuppressionPresets.map((preset) => ( + + } + > + + + ))} + + )} ); }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 8cc76803b..0993a1511 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -9,6 +9,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject } from "rxjs"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; +import type { RNNoiseSuppressionPreset } from "../audio/rnnoiseTypes"; import { type Behavior } from "../state/Behavior"; import { useBehavior } from "../useBehavior"; @@ -101,6 +102,11 @@ export const rnnoiseNoiseSuppression = new Setting( "rnnoise-noise-suppression", false, ); +export const rnnoiseNoiseSuppressionPreset = + new Setting( + "rnnoise-noise-suppression-preset", + "conservative", + ); export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index d121d3a2d..9bb79b777 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -27,7 +27,11 @@ import { import { Publisher } from "./Publisher"; import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; -import { rnnoiseNoiseSuppression } from "../../../settings/settings"; +import { + rnnoiseNoiseSuppression, + rnnoiseNoiseSuppressionPreset, +} from "../../../settings/settings"; +import type { RNNoiseProcessor } from "../../../audio/RNNoiseProcessor"; let scope: ObservableScope; @@ -112,6 +116,7 @@ let createTrackLock: Promise; beforeEach(() => { rnnoiseNoiseSuppression.setValue(false); + rnnoiseNoiseSuppressionPreset.setValue("conservative"); trackPublications = []; audioEnabled$ = new BehaviorSubject(false); videoEnabled$ = new BehaviorSubject(false); @@ -372,6 +377,7 @@ describe("Publisher", () => { afterEach(() => { vi.unstubAllGlobals(); rnnoiseNoiseSuppression.setValue(false); + rnnoiseNoiseSuppressionPreset.setValue("conservative"); }); it("enabling setting applies RNNoise processor on microphone track", async () => { @@ -443,6 +449,32 @@ describe("Publisher", () => { }), ); }); + + it("updates active RNNoise processor preset when preset setting changes", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { getProcessor: () => unknown }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + + const processor = micTrack.getProcessor() as RNNoiseProcessor; + const setPresetSpy = vi.spyOn(processor, "setPreset"); + + rnnoiseNoiseSuppressionPreset.setValue("strong"); + await flushPromises(); + + expect(setPresetSpy).toHaveBeenCalledWith("strong"); + }); }); }); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 50e27b157..8568bce48 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -41,7 +41,11 @@ import { RNNoiseProcessor, supportsRNNoiseProcessor, } from "../../../audio/RNNoiseProcessor.ts"; -import { rnnoiseNoiseSuppression } from "../../../settings/settings.ts"; +import { + rnnoiseNoiseSuppression, + rnnoiseNoiseSuppressionPreset, +} from "../../../settings/settings.ts"; +import type { RNNoiseSuppressionPreset } from "../../../audio/rnnoiseTypes.ts"; /** * A wrapper for a Connection object. @@ -447,17 +451,29 @@ export class Publisher { null, ); - combineLatest([microphoneTrack$, rnnoiseNoiseSuppression.value$]) + combineLatest([ + microphoneTrack$, + rnnoiseNoiseSuppression.value$, + rnnoiseNoiseSuppressionPreset.value$, + ]) .pipe( scope.bind(), - distinctUntilChanged(([aTrack, aEnabled], [bTrack, bEnabled]) => { - return aTrack === bTrack && aEnabled === bEnabled; - }), + distinctUntilChanged( + ([aTrack, aEnabled, aPreset], [bTrack, bEnabled, bPreset]) => { + return ( + aTrack === bTrack && aEnabled === bEnabled && aPreset === bPreset + ); + }, + ), ) - .subscribe(([microphoneTrack, rnnoiseEnabled]) => { + .subscribe(([microphoneTrack, rnnoiseEnabled, rnnoisePreset]) => { if (!microphoneTrack || !supportsRNNoiseProcessor()) return; - void this.syncRNNoiseProcessor(microphoneTrack, rnnoiseEnabled); + void this.syncRNNoiseProcessor( + microphoneTrack, + rnnoiseEnabled, + rnnoisePreset, + ); }); } @@ -492,14 +508,25 @@ export class Publisher { private async syncRNNoiseProcessor( microphoneTrack: LocalAudioTrack, rnnoiseEnabled: boolean, + rnnoisePreset: RNNoiseSuppressionPreset, ): Promise { try { const processor = microphoneTrack.getProcessor(); const rnnoiseActive = processor?.name === "rnnoise-noise-suppression"; + const rnnoiseProcessor = + processor instanceof RNNoiseProcessor ? processor : undefined; - if (rnnoiseEnabled && !rnnoiseActive) { - await microphoneTrack.setProcessor(new RNNoiseProcessor()); - } else if (!rnnoiseEnabled && rnnoiseActive) { + if (rnnoiseEnabled) { + if (rnnoiseProcessor) { + rnnoiseProcessor.setPreset(rnnoisePreset); + return; + } + + if (rnnoiseActive) { + await microphoneTrack.stopProcessor(); + } + await microphoneTrack.setProcessor(new RNNoiseProcessor(rnnoisePreset)); + } else if (rnnoiseActive) { await microphoneTrack.stopProcessor(); } } catch (e) { From e54d1422488301369e1e9863de4cf820a80bdbc9 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 01:23:16 +0200 Subject: [PATCH 05/28] fix(audio): serialize RNNoise toggle restart and processor sync Signed-off-by: LucaPisl --- .../localMember/Publisher.test.ts | 35 +++++++++ .../CallViewModel/localMember/Publisher.ts | 71 +++++++++++++------ 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 9bb79b777..2ce3abf53 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -450,6 +450,41 @@ describe("Publisher", () => { ); }); + it("stops RNNoise processor before restarting microphone track when disabling RNNoise", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + stopProcessor: () => void; + restartTrack: (...args: unknown[]) => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + + vi.mocked(micTrack.stopProcessor).mockClear(); + vi.mocked(micTrack.restartTrack).mockClear(); + + rnnoiseNoiseSuppression.setValue(false); + await flushPromises(); + + expect(micTrack.stopProcessor).toHaveBeenCalledOnce(); + expect(micTrack.restartTrack).toHaveBeenCalledOnce(); + expect( + vi.mocked(micTrack.stopProcessor).mock.invocationCallOrder[0], + ).toBeLessThan( + vi.mocked(micTrack.restartTrack).mock.invocationCallOrder[0], + ); + }); + it("updates active RNNoise processor preset when preset setting changes", async () => { const micTrack = createMockLocalTrack( Track.Source.Microphone, diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 8568bce48..de72a22c2 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -61,6 +61,7 @@ export class Publisher { public shouldPublish = false; private readonly scope = new ObservableScope(); + private rnnoiseOperationQueue: Promise = Promise.resolve(); /** * Creates a new Publisher. @@ -459,21 +460,21 @@ export class Publisher { .pipe( scope.bind(), distinctUntilChanged( - ([aTrack, aEnabled, aPreset], [bTrack, bEnabled, bPreset]) => { - return ( - aTrack === bTrack && aEnabled === bEnabled && aPreset === bPreset - ); + ([aTrack, _aEnabled, aPreset], [bTrack, _bEnabled, bPreset]) => { + return aTrack === bTrack && aPreset === bPreset; }, ), ) .subscribe(([microphoneTrack, rnnoiseEnabled, rnnoisePreset]) => { if (!microphoneTrack || !supportsRNNoiseProcessor()) return; - void this.syncRNNoiseProcessor( - microphoneTrack, - rnnoiseEnabled, - rnnoisePreset, - ); + this.enqueueRNNoiseOperation(async () => { + await this.syncRNNoiseProcessor( + microphoneTrack, + rnnoiseEnabled, + rnnoisePreset, + ); + }); }); } @@ -490,21 +491,49 @@ export class Publisher { )?.audioTrack; if (!audioTrack) return; - const { echoCancellation = true, noiseSuppression = true } = - getUrlParams(); - - void audioTrack - .restartTrack({ - deviceId: devices.audioInput.selected$.value?.id, - echoCancellation, - noiseSuppression: noiseSuppression && !rnnoiseEnabled, - }) - .catch((e) => { - this.logger.error("Failed to restart microphone track", e); - }); + this.enqueueRNNoiseOperation(async () => { + await this.restartMicrophoneTrackForNoiseSuppressionPolicy( + audioTrack, + devices, + rnnoiseEnabled, + ); + await this.syncRNNoiseProcessor( + audioTrack, + rnnoiseEnabled, + rnnoiseNoiseSuppressionPreset.getValue(), + ); + }); }); } + private enqueueRNNoiseOperation(operation: () => Promise): void { + this.rnnoiseOperationQueue = this.rnnoiseOperationQueue.then(async () => { + try { + await operation(); + } catch (e) { + this.logger.error("Failed to process RNNoise operation", e); + } + }); + } + + private async restartMicrophoneTrackForNoiseSuppressionPolicy( + audioTrack: LocalAudioTrack, + devices: MediaDevices, + rnnoiseEnabled: boolean, + ): Promise { + const activeProcessor = audioTrack.getProcessor(); + if (activeProcessor?.name === "rnnoise-noise-suppression") { + await audioTrack.stopProcessor(); + } + + const { echoCancellation = true, noiseSuppression = true } = getUrlParams(); + await audioTrack.restartTrack({ + deviceId: devices.audioInput.selected$.value?.id, + echoCancellation, + noiseSuppression: noiseSuppression && !rnnoiseEnabled, + }); + } + private async syncRNNoiseProcessor( microphoneTrack: LocalAudioTrack, rnnoiseEnabled: boolean, From be0380bf7c3316711b42c567498f23c523808c0d Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 02:48:29 +0200 Subject: [PATCH 06/28] fix(audio): bypass RNNoise on non-48kHz sample rates Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 152 ++++++++++++++++++++++++++++- src/audio/RNNoiseProcessor.ts | 87 ++++++++++++++--- 2 files changed, 225 insertions(+), 14 deletions(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index dcdaf2675..b5fbbac3c 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -7,8 +7,13 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { Track } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; -import { RNNoiseProcessor, supportsRNNoiseProcessor } from "./RNNoiseProcessor"; +import { + createRNNoiseWorkletCodeForTesting, + RNNoiseProcessor, + supportsRNNoiseProcessor, +} from "./RNNoiseProcessor"; vi.mock("@jitsi/rnnoise-wasm/dist/rnnoise-sync.js?raw", () => ({ default: @@ -27,7 +32,7 @@ type TestContext = { track: MediaStreamTrack; }; -function createTestContext(): TestContext { +function createTestContext(sampleRate = 48000): TestContext { const processedTrack = { id: "processed-track" } as MediaStreamTrack; const sourceNode = { connect: vi.fn(), @@ -50,6 +55,7 @@ function createTestContext(): TestContext { const createSourceNode = vi.fn().mockReturnValue(sourceNode); const createDestinationNode = vi.fn().mockReturnValue(destinationNode); const audioContext = { + sampleRate, audioWorklet: { addModule }, createMediaStreamSource: createSourceNode, createMediaStreamDestination: createDestinationNode, @@ -72,6 +78,65 @@ function createTestContext(): TestContext { }; } +function getGeneratedWorkletCode(): string { + return createRNNoiseWorkletCodeForTesting( + "function createRNNWasmModuleSync(){}; export default createRNNWasmModuleSync;", + ); +} + +function instantiateWorkletProcessor(workletCode: string): { + process: ( + inputs: Float32Array[][], + outputs: Float32Array[][], + params?: Record, + ) => boolean; +} { + let ProcessorCtor: + | (new () => { + process: ( + inputs: Float32Array[][], + outputs: Float32Array[][], + params?: Record, + ) => boolean; + }) + | undefined; + + class TestAudioWorkletProcessor { + public readonly port = { + postMessage: vi.fn(), + onmessage: null as ((event: MessageEvent) => void) | null, + }; + } + + const registerProcessor = vi.fn( + ( + _name: string, + ctor: new () => { + process: ( + inputs: Float32Array[][], + outputs: Float32Array[][], + params?: Record, + ) => boolean; + }, + ) => { + ProcessorCtor = ctor; + }, + ); + + const runWorkletModule = new Function( + "AudioWorkletProcessor", + "registerProcessor", + workletCode, + ); + runWorkletModule(TestAudioWorkletProcessor, registerProcessor); + + if (!ProcessorCtor) { + throw new Error("Expected worklet processor to be registered."); + } + + return new ProcessorCtor(); +} + describe("RNNoiseProcessor", () => { beforeEach(() => { vi.stubGlobal( @@ -217,4 +282,87 @@ describe("RNNoiseProcessor", () => { preset: "strong", }); }); + + it("bypasses RNNoise for unsupported audio context sample rates", async () => { + const t = createTestContext(44100); + const workletCtor = vi.fn().mockReturnValue(t.workletNode); + const warningSpy = vi.spyOn(logger, "warn"); + vi.stubGlobal("AudioWorkletNode", workletCtor); + const processor = new RNNoiseProcessor(); + + await expect( + processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }), + ).rejects.toThrow("48000Hz"); + + expect(warningSpy).toHaveBeenCalledOnce(); + expect(t.addModule).not.toHaveBeenCalled(); + expect(workletCtor).not.toHaveBeenCalled(); + expect(processor.processedTrack).toBeUndefined(); + }); + + it("releases the worklet blob URL when worklet registration fails", async () => { + const t = createTestContext(); + const workletCtor = vi.fn().mockReturnValue(t.workletNode); + const addModuleError = new Error("Failed to register worklet module"); + t.addModule.mockRejectedValueOnce(addModuleError); + vi.stubGlobal("AudioWorkletNode", workletCtor); + const processor = new RNNoiseProcessor(); + + await expect( + processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }), + ).rejects.toThrow(addModuleError); + + expect(workletCtor).not.toHaveBeenCalled(); + expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:rnnoise"); + }); + + it("restarts with the last known audio context when restart omits audioContext", async () => { + const t = createTestContext(); + const workletCtor = vi.fn().mockReturnValue(t.workletNode); + vi.stubGlobal("AudioWorkletNode", workletCtor); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + const restartedTrack = { + id: "restarted-input-track", + kind: Track.Kind.Audio, + } as MediaStreamTrack; + await processor.restart({ + kind: Track.Kind.Audio, + track: restartedTrack, + // LiveKit restart paths can omit audioContext. + audioContext: undefined as unknown as AudioContext, + }); + + expect(t.addModule).toHaveBeenCalledOnce(); + expect(t.createSourceNode).toHaveBeenCalledTimes(2); + expect(workletCtor).toHaveBeenCalledTimes(2); + }); + + it("deterministically downmixes stereo input to mono in the worklet passthrough path", () => { + const workletCode = getGeneratedWorkletCode(); + const worklet = instantiateWorkletProcessor(workletCode); + const left = new Float32Array([1, -1, 0.5, 0]); + const right = new Float32Array([0, 1, -0.5, 0.5]); + const output = new Float32Array(left.length); + + const keepProcessing = worklet.process([[left, right]], [[output]], {}); + + expect(keepProcessing).toBe(true); + expect(output).toEqual(new Float32Array([0.5, 0, 0, 0.25])); + expect(output).toHaveLength(left.length); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index b013a2ffe..9747d9836 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -5,6 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { logger } from "matrix-js-sdk/lib/logger"; + import type { AudioProcessorOptions, Track, @@ -16,9 +18,28 @@ import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; * The number of samples per frame expected by RNNoise (at 48kHz = 10ms). */ const RNNOISE_SAMPLE_LENGTH = 480; +const RNNOISE_REQUIRED_SAMPLE_RATE = 48000; const RNNOISE_WORKLET_NAME = "rnnoise-processor"; const DEFAULT_RNNOISE_PRESET: RNNoiseSuppressionPreset = "conservative"; const loadedAudioWorklets = new WeakSet(); +const warnedUnsupportedSampleRates = new Set(); + +function createUnsupportedSampleRateError(sampleRate: number): Error { + return new Error( + `RNNoise requires an AudioContext sample rate of ${RNNOISE_REQUIRED_SAMPLE_RATE}Hz (received ${sampleRate}Hz).`, + ); +} + +function warnUnsupportedSampleRate(sampleRate: number): void { + if (warnedUnsupportedSampleRates.has(sampleRate)) { + return; + } + + warnedUnsupportedSampleRates.add(sampleRate); + logger.warn( + `Skipping RNNoise because AudioContext sample rate is ${sampleRate}Hz (expected ${RNNOISE_REQUIRED_SAMPLE_RATE}Hz).`, + ); +} /** * Whether the current runtime supports the required APIs for RNNoise. @@ -51,7 +72,7 @@ ${patched} const FRAME_SIZE = ${RNNOISE_SAMPLE_LENGTH}; const RING_SIZE = FRAME_SIZE * 3; // Enough headroom for buffering -const SAMPLE_RATE = 48000; +const SAMPLE_RATE = ${RNNOISE_REQUIRED_SAMPLE_RATE}; const DEFAULT_PRESET = "${DEFAULT_RNNOISE_PRESET}"; const PRESETS = { conservative: { @@ -219,25 +240,41 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._outW = (this._outW + FRAME_SIZE) % RING_SIZE; } + _mixInputChannels(inputChannels, sampleIndex, channelCount) { + let mixed = 0; + for (let i = 0; i < channelCount; i++) { + const channel = inputChannels[i]; + mixed += channel ? (channel[sampleIndex] ?? 0) : 0; + } + return mixed / channelCount; + } + process(inputs, outputs) { if (this._destroyed) return false; - const input = inputs[0]?.[0]; + const inputChannels = inputs[0]; const output = outputs[0]?.[0]; - if (!input || !output) return true; + if (!inputChannels?.length || !output) return true; + + const blockSize = output.length; + const channelCount = inputChannels.length; if (!this._ready) { - // Pass through until RNNoise is ready - output.set(input); + // Pass through until RNNoise is ready, with deterministic stereo downmix. + for (let i = 0; i < blockSize; i++) { + output[i] = this._mixInputChannels(inputChannels, i, channelCount); + } return true; } - const blockSize = input.length; - // Write input samples to the input ring buffer for (let i = 0; i < blockSize; i++) { - this._inBuf[this._inW] = input[i]; + this._inBuf[this._inW] = this._mixInputChannels( + inputChannels, + i, + channelCount, + ); this._inW = (this._inW + 1) % RING_SIZE; } @@ -267,6 +304,12 @@ registerProcessor('${RNNOISE_WORKLET_NAME}', RNNoiseWorkletProcessor); `; } +export function createRNNoiseWorkletCodeForTesting( + rnnoiseModuleCode: string, +): string { + return createWorkletCode(rnnoiseModuleCode); +} + /** * A LiveKit TrackProcessor that applies RNNoise-based noise suppression * to a local audio track via an AudioWorklet. @@ -287,6 +330,7 @@ export class RNNoiseProcessor implements TrackProcessor< private blobUrl?: string; private destroyed = false; private preset: RNNoiseSuppressionPreset; + private lastAudioContext?: AudioContext; public constructor( preset: RNNoiseSuppressionPreset = DEFAULT_RNNOISE_PRESET, @@ -310,16 +354,27 @@ export class RNNoiseProcessor implements TrackProcessor< // Create a Blob URL for the AudioWorklet module. const blob = new Blob([workletCode], { type: "text/javascript" }); - this.blobUrl = URL.createObjectURL(blob); + const blobUrl = URL.createObjectURL(blob); - await audioContext.audioWorklet.addModule(this.blobUrl); - loadedAudioWorklets.add(audioContext); + try { + await audioContext.audioWorklet.addModule(blobUrl); + loadedAudioWorklets.add(audioContext); + this.blobUrl = blobUrl; + } catch (e) { + URL.revokeObjectURL(blobUrl); + throw e; + } } public async init(opts: AudioProcessorOptions): Promise { this.destroyed = false; const { audioContext, track } = opts; + if (audioContext.sampleRate !== RNNOISE_REQUIRED_SAMPLE_RATE) { + warnUnsupportedSampleRate(audioContext.sampleRate); + throw createUnsupportedSampleRateError(audioContext.sampleRate); + } + await this.ensureWorkletRegistered(audioContext); // Build the audio processing graph: @@ -345,11 +400,19 @@ export class RNNoiseProcessor implements TrackProcessor< this.destinationNode = destinationNode; this.workletNode.port.postMessage({ type: "preset", preset: this.preset }); this.processedTrack = destinationNode.stream.getAudioTracks()[0]; + this.lastAudioContext = audioContext; } public async restart(opts: AudioProcessorOptions): Promise { + const audioContext = opts.audioContext ?? this.lastAudioContext; + if (!audioContext) { + throw new Error( + "RNNoise restart requires an AudioContext when no previous context has been initialized.", + ); + } + await this.destroy(); - await this.init(opts); + await this.init({ ...opts, audioContext }); } public async destroy(): Promise { From 9755519669374dd6afa0b0159becbb86de19192c Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 02:48:33 +0200 Subject: [PATCH 07/28] test(audio): cover RNNoise worklet load failure fallback Signed-off-by: LucaPisl --- .../localMember/Publisher.test.ts | 39 +++++++++++++++++++ .../CallViewModel/localMember/Publisher.ts | 6 +++ 2 files changed, 45 insertions(+) diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 2ce3abf53..1210e8937 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -426,6 +426,45 @@ describe("Publisher", () => { expect(micTrack.stopProcessor).toHaveBeenCalledOnce(); }); + it("auto-disables RNNoise when processor setup fails and falls back to native noise suppression", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + setProcessor: (...args: unknown[]) => Promise; + restartTrack: (...args: unknown[]) => Promise; + }; + const processorError = new Error("RNNoise setup failed"); + vi.mocked(micTrack.setProcessor).mockRejectedValueOnce(processorError); + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + + // The operation queue serializes restart/sync jobs. Flush enough microtasks + // to allow the fallback toggle and restart to complete. + for (let i = 0; i < 5; i++) { + await flushPromises(); + } + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + expect(rnnoiseNoiseSuppression.getValue()).toBe(false); + expect(micTrack.restartTrack).toHaveBeenCalled(); + const restartCalls = vi.mocked(micTrack.restartTrack).mock.calls; + const finalRestartConfig = restartCalls[restartCalls.length - 1]?.[0] as { + noiseSuppression?: boolean; + }; + expect(finalRestartConfig).toEqual( + expect.objectContaining({ noiseSuppression: true }), + ); + }); + it("restarts microphone track with native noise suppression disabled when RNNoise is enabled", async () => { const micTrack = createMockLocalTrack( Track.Source.Microphone, diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index de72a22c2..223742430 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -560,6 +560,12 @@ export class Publisher { } } catch (e) { this.logger.error("Failed to apply RNNoise microphone processor", e); + if (rnnoiseEnabled && rnnoiseNoiseSuppression.getValue()) { + this.logger.warn( + "Disabling RNNoise setting after processor setup failure", + ); + rnnoiseNoiseSuppression.setValue(false); + } } } } From 68c1b481bdd85a22182d632e4a06f90bfa268332 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 02:48:40 +0200 Subject: [PATCH 08/28] test(settings): add RNNoise SettingsModal coverage Signed-off-by: LucaPisl --- src/settings/SettingsModal.test.tsx | 181 ++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 src/settings/SettingsModal.test.tsx diff --git a/src/settings/SettingsModal.test.tsx b/src/settings/SettingsModal.test.tsx new file mode 100644 index 000000000..948110e1b --- /dev/null +++ b/src/settings/SettingsModal.test.tsx @@ -0,0 +1,181 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, beforeEach, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { TooltipProvider } from "@vector-im/compound-web"; + +import type { MatrixClient } from "matrix-js-sdk"; +import type { ReactNode } from "react"; +import { SettingsModal } from "./SettingsModal"; +import { + rnnoiseNoiseSuppression, + rnnoiseNoiseSuppressionPreset, +} from "./settings"; +import { supportsRNNoiseProcessor } from "../audio/RNNoiseProcessor"; + +const { mockRequestDeviceNames } = vi.hoisted(() => ({ + mockRequestDeviceNames: vi.fn(), +})); + +vi.mock("../audio/RNNoiseProcessor", async () => { + const actual = await vi.importActual("../audio/RNNoiseProcessor"); + + return { + ...actual, + supportsRNNoiseProcessor: vi.fn(() => true), + }; +}); + +vi.mock("../Modal", () => ({ + Modal: ({ + open, + children, + }: { + open: boolean; + children: ReactNode; + }): ReactNode => (open ?
{children}
: null), +})); + +vi.mock("../tabs/Tabs", () => ({ + TabContainer: ({ + tab, + tabs, + }: { + tab: string; + tabs: { key: string; content: ReactNode }[]; + }): ReactNode => ( +
{tabs.find((candidate) => candidate.key === tab)?.content}
+ ), +})); + +vi.mock("../MediaDevicesContext", () => ({ + useMediaDevices: (): { + requestDeviceNames: typeof mockRequestDeviceNames; + audioInput: object; + audioOutput: object; + videoInput: object; + } => ({ + requestDeviceNames: mockRequestDeviceNames, + audioInput: {}, + audioOutput: {}, + videoInput: {}, + }), +})); + +vi.mock("./DeviceSelection", () => ({ + DeviceSelection: (): ReactNode =>
, +})); + +vi.mock("../livekit/TrackProcessorContext", () => ({ + useTrackProcessor: (): { supported: boolean; processor: undefined } => ({ + supported: true, + processor: undefined, + }), +})); + +vi.mock("./submit-rageshake", () => ({ + useSubmitRageshake: (): { + submitRageshake: ReturnType; + sending: boolean; + sent: boolean; + error: undefined; + available: boolean; + } => ({ + submitRageshake: vi.fn(), + sending: false, + sent: false, + error: undefined, + available: false, + }), +})); + +vi.mock("../UrlParams", async () => { + const actual = await vi.importActual("../UrlParams"); + return { + ...actual, + useUrlParams: (): { controlledAudioDevices: boolean } => ({ + controlledAudioDevices: false, + }), + }; +}); + +function renderSettingsModal(): void { + render( + + + , + ); +} + +describe("SettingsModal RNNoise controls", () => { + beforeEach(() => { + vi.stubGlobal( + "ResizeObserver", + class ResizeObserver { + public observe(): void {} + public unobserve(): void {} + public disconnect(): void {} + }, + ); + localStorage.clear(); + mockRequestDeviceNames.mockClear(); + rnnoiseNoiseSuppressionPreset.setValue("conservative"); + rnnoiseNoiseSuppression.setValue(false); + vi.mocked(supportsRNNoiseProcessor).mockReturnValue(true); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("renders the RNNoise checkbox in the audio tab", () => { + renderSettingsModal(); + + expect( + screen.getByLabelText("Enable enhanced noise suppression (RNNoise)"), + ).toBeInTheDocument(); + expect(mockRequestDeviceNames).toHaveBeenCalledOnce(); + }); + + it("disables RNNoise when AudioWorklet support is unavailable", () => { + vi.mocked(supportsRNNoiseProcessor).mockReturnValue(false); + + renderSettingsModal(); + + expect( + screen.getByLabelText("Enable enhanced noise suppression (RNNoise)"), + ).toBeDisabled(); + expect( + screen.getByText( + "(Enhanced noise suppression is not supported by this browser.)", + ), + ).toBeInTheDocument(); + }); + + it("persists RNNoise setting when toggled", async () => { + const user = userEvent.setup(); + renderSettingsModal(); + + const checkbox = screen.getByLabelText( + "Enable enhanced noise suppression (RNNoise)", + ); + await user.click(checkbox); + + expect(rnnoiseNoiseSuppression.getValue()).toBe(true); + expect( + localStorage.getItem("matrix-setting-rnnoise-noise-suppression"), + ).toBe("true"); + }); +}); From dfae961f3cd6f28cc5ca8c1383eaa5aec63f8a68 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 02:49:24 +0200 Subject: [PATCH 09/28] test(audio): specify and validate RNNoise downmix policy Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 16 ++++++++++++++++ src/audio/RNNoiseProcessor.ts | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index b5fbbac3c..d4c2dcb31 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -365,4 +365,20 @@ describe("RNNoiseProcessor", () => { expect(output).toEqual(new Float32Array([0.5, 0, 0, 0.25])); expect(output).toHaveLength(left.length); }); + + it("downmixes all input channels by averaging each sample", () => { + const workletCode = getGeneratedWorkletCode(); + const worklet = instantiateWorkletProcessor(workletCode); + const first = new Float32Array([0.6, -0.3, 0.9]); + const second = new Float32Array([0.3, 0.3, -0.3]); + const third = new Float32Array([0, 0.6, 0]); + const output = new Float32Array(first.length); + + worklet.process([[first, second, third]], [[output]], {}); + + expect(output[0]).toBeCloseTo(0.3, 6); + expect(output[1]).toBeCloseTo(0.2, 6); + expect(output[2]).toBeCloseTo(0.2, 6); + expect(output).toHaveLength(first.length); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 9747d9836..66cbab58e 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -241,6 +241,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { } _mixInputChannels(inputChannels, sampleIndex, channelCount) { + // RNNoise is mono-only; average all channels for deterministic downmixing. let mixed = 0; for (let i = 0; i < channelCount; i++) { const channel = inputChannels[i]; @@ -261,7 +262,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { const channelCount = inputChannels.length; if (!this._ready) { - // Pass through until RNNoise is ready, with deterministic stereo downmix. + // Pass through until RNNoise is ready, with deterministic mono downmix. for (let i = 0; i < blockSize; i++) { output[i] = this._mixInputChannels(inputChannels, i, channelCount); } From ebf2cbb37b0876c27e94defe2fa5ab4955a52ba7 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 02:55:27 +0200 Subject: [PATCH 10/28] test(playwright): add RNNoise rejoin stability coverage Signed-off-by: LucaPisl --- playwright/spa-call-sticky.spec.ts | 68 ++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/playwright/spa-call-sticky.spec.ts b/playwright/spa-call-sticky.spec.ts index 246b4a73a..c97d02144 100644 --- a/playwright/spa-call-sticky.spec.ts +++ b/playwright/spa-call-sticky.spec.ts @@ -114,6 +114,44 @@ test("One to One rejoin after improper leave does not crash EC", async ({ await expect(guestPage.getByTestId("videoTile")).toHaveCount(2); }); +test("One to One rejoin after improper leave stays stable with RNNoise enabled", async ({ + browser, + page, + browserName, +}) => { + const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); + + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); + test.skip( + !rnnoiseSupported, + "RNNoise is not supported in this browser environment", + ); + + await expect + .poll(async () => + guestPage.evaluate(() => + localStorage.getItem("matrix-setting-rnnoise-noise-suppression"), + ), + ) + .toBe("true"); + + await guestPage.reload(); + await expect(guestPage.getByTestId("lobby_joinCall")).toBeVisible(); + await guestPage.getByTestId("lobby_joinCall").click(); + + // Rejoin after abrupt disconnect should remain stable with RNNoise enabled. + await expect(page.getByTestId("videoTile")).toHaveCount(3); + await expect(guestPage.getByTestId("videoTile")).toHaveCount(2); + await expect( + guestPage.getByRole("button", { name: "Mute microphone" }), + ).toBeVisible(); + + await expectRNNoiseEnabledInSettings(guestPage); +}); + function isStickySend(url: string): boolean { return !!new URL(url).searchParams.get( "org.matrix.msc4354.sticky_duration_ms", @@ -133,3 +171,33 @@ async function interceptEventSend( }, ); } + +async function enableRNNoiseInSettings(page: Page): Promise { + await page.getByRole("button", { name: "Settings" }).click(); + await page.getByRole("tab", { name: "Audio" }).click(); + + const rnnoiseToggle = page.getByLabel( + "Enable enhanced noise suppression (RNNoise)", + ); + await expect(rnnoiseToggle).toBeVisible(); + + const supported = await rnnoiseToggle.isEnabled(); + if (supported && !(await rnnoiseToggle.isChecked())) { + await rnnoiseToggle.check(); + } + + await page.getByTestId("modal_close").click(); + return supported; +} + +async function expectRNNoiseEnabledInSettings(page: Page): Promise { + await page.getByRole("button", { name: "Settings" }).click(); + await page.getByRole("tab", { name: "Audio" }).click(); + + const rnnoiseToggle = page.getByLabel( + "Enable enhanced noise suppression (RNNoise)", + ); + await expect(rnnoiseToggle).toBeChecked(); + + await page.getByTestId("modal_close").click(); +} From 2044d9520f8033d5a7204b2578089d2c12e44ea1 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 04:00:36 +0200 Subject: [PATCH 11/28] fix(audio): fail open to native suppression when RNNoise unsupported Signed-off-by: LucaPisl --- src/audio/noiseSuppressionPolicy.test.ts | 49 +++++++++++++++++++ src/audio/noiseSuppressionPolicy.ts | 26 ++++++++++ src/settings/SettingsModal.test.tsx | 9 ++-- src/settings/SettingsModal.tsx | 5 +- src/state/CallViewModel/CallViewModel.ts | 9 +++- .../localMember/Publisher.test.ts | 31 ++++++++++++ .../CallViewModel/localMember/Publisher.ts | 10 +++- 7 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 src/audio/noiseSuppressionPolicy.test.ts create mode 100644 src/audio/noiseSuppressionPolicy.ts diff --git a/src/audio/noiseSuppressionPolicy.test.ts b/src/audio/noiseSuppressionPolicy.test.ts new file mode 100644 index 000000000..5ad2deaf3 --- /dev/null +++ b/src/audio/noiseSuppressionPolicy.test.ts @@ -0,0 +1,49 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { shouldEnableNativeNoiseSuppression } from "./noiseSuppressionPolicy"; + +describe("shouldEnableNativeNoiseSuppression", () => { + it("keeps native suppression enabled when RNNoise is enabled but unsupported", () => { + expect( + shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: true, + rnnoiseEnabled: true, + rnnoiseSupported: false, + }), + ).toBe(true); + }); + + it("disables native suppression only when RNNoise is enabled and supported", () => { + expect( + shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: true, + rnnoiseEnabled: true, + rnnoiseSupported: true, + }), + ).toBe(false); + }); + + it("keeps native suppression disabled when explicitly disabled by URL", () => { + expect( + shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: false, + rnnoiseEnabled: false, + rnnoiseSupported: false, + }), + ).toBe(false); + expect( + shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: false, + rnnoiseEnabled: true, + rnnoiseSupported: false, + }), + ).toBe(false); + }); +}); diff --git a/src/audio/noiseSuppressionPolicy.ts b/src/audio/noiseSuppressionPolicy.ts new file mode 100644 index 000000000..93ec150be --- /dev/null +++ b/src/audio/noiseSuppressionPolicy.ts @@ -0,0 +1,26 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +interface NativeNoiseSuppressionPolicyOptions { + urlNoiseSuppression: boolean | undefined; + rnnoiseEnabled: boolean; + rnnoiseSupported: boolean; +} + +/** + * Resolves whether native browser noise suppression should be enabled. + * + * Native suppression should only be disabled when RNNoise is both enabled and + * actually supported at runtime. + */ +export function shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression, + rnnoiseEnabled, + rnnoiseSupported, +}: NativeNoiseSuppressionPolicyOptions): boolean { + return (urlNoiseSuppression ?? true) && !(rnnoiseEnabled && rnnoiseSupported); +} diff --git a/src/settings/SettingsModal.test.tsx b/src/settings/SettingsModal.test.tsx index 948110e1b..b44d9cb3d 100644 --- a/src/settings/SettingsModal.test.tsx +++ b/src/settings/SettingsModal.test.tsx @@ -151,12 +151,15 @@ describe("SettingsModal RNNoise controls", () => { it("disables RNNoise when AudioWorklet support is unavailable", () => { vi.mocked(supportsRNNoiseProcessor).mockReturnValue(false); + rnnoiseNoiseSuppression.setValue(true); renderSettingsModal(); - expect( - screen.getByLabelText("Enable enhanced noise suppression (RNNoise)"), - ).toBeDisabled(); + const checkbox = screen.getByLabelText( + "Enable enhanced noise suppression (RNNoise)", + ); + expect(checkbox).toBeDisabled(); + expect(checkbox).not.toBeChecked(); expect( screen.getByText( "(Enhanced noise suppression is not supported by this browser.)", diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 58d19596f..659ec8f55 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -138,6 +138,7 @@ export const SettingsModal: FC = ({ balanced: t("settings.audio_tab.rnnoise_preset_balanced"), strong: t("settings.audio_tab.rnnoise_preset_strong"), }; + const effectiveRnnoiseEnabled = supported && !!rnnoiseEnabled; return ( <> @@ -150,12 +151,12 @@ export const SettingsModal: FC = ({ supported ? "" : t("settings.audio_tab.rnnoise_not_supported") } type="checkbox" - checked={!!rnnoiseEnabled} + checked={effectiveRnnoiseEnabled} onChange={(e): void => setRnnoiseEnabled(e.target.checked)} disabled={!supported} /> - {rnnoiseEnabled && ( + {effectiveRnnoiseEnabled && ( <>

{t("settings.audio_tab.rnnoise_preset_description")}

{rnnoiseSuppressionPresets.map((preset) => ( diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index b95550fb4..9e9c4ef46 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -64,6 +64,8 @@ import { rnnoiseNoiseSuppression, showReactions, } from "../../settings/settings"; +import { shouldEnableNativeNoiseSuppression } from "../../audio/noiseSuppressionPolicy"; +import { supportsRNNoiseProcessor } from "../../audio/RNNoiseProcessor"; import { isFirefox } from "../../Platform"; import { setPipEnabled$ } from "../../controls"; import { TileStore } from "../TileStore"; @@ -481,8 +483,11 @@ export function createCallViewModel$( getUrlParams().controlledAudioDevices, options.livekitRoomFactory, getUrlParams().echoCancellation, - (getUrlParams().noiseSuppression ?? true) && - !rnnoiseNoiseSuppression.getValue(), + shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: getUrlParams().noiseSuppression, + rnnoiseEnabled: rnnoiseNoiseSuppression.getValue(), + rnnoiseSupported: supportsRNNoiseProcessor(), + }), ); const connectionManager = createConnectionManager$({ diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 1210e8937..fc7c41609 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -489,6 +489,37 @@ describe("Publisher", () => { ); }); + it("keeps native noise suppression enabled and skips processor when RNNoise is unsupported", async () => { + vi.stubGlobal("AudioWorkletNode", undefined); + vi.stubGlobal("MediaStreamAudioDestinationNode", undefined); + vi.stubGlobal("MediaStreamAudioSourceNode", undefined); + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + restartTrack: (...args: unknown[]) => void; + setProcessor: (...args: unknown[]) => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + + expect(micTrack.setProcessor).not.toHaveBeenCalled(); + expect(micTrack.restartTrack).toHaveBeenCalledWith( + expect.objectContaining({ + noiseSuppression: true, + }), + ); + }); + it("stops RNNoise processor before restarting microphone track when disabling RNNoise", async () => { const micTrack = createMockLocalTrack( Track.Source.Microphone, diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 223742430..bea386c86 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -41,6 +41,7 @@ import { RNNoiseProcessor, supportsRNNoiseProcessor, } from "../../../audio/RNNoiseProcessor.ts"; +import { shouldEnableNativeNoiseSuppression } from "../../../audio/noiseSuppressionPolicy.ts"; import { rnnoiseNoiseSuppression, rnnoiseNoiseSuppressionPreset, @@ -491,6 +492,7 @@ export class Publisher { )?.audioTrack; if (!audioTrack) return; + const rnnoiseSupported = supportsRNNoiseProcessor(); this.enqueueRNNoiseOperation(async () => { await this.restartMicrophoneTrackForNoiseSuppressionPolicy( audioTrack, @@ -499,7 +501,7 @@ export class Publisher { ); await this.syncRNNoiseProcessor( audioTrack, - rnnoiseEnabled, + rnnoiseEnabled && rnnoiseSupported, rnnoiseNoiseSuppressionPreset.getValue(), ); }); @@ -530,7 +532,11 @@ export class Publisher { await audioTrack.restartTrack({ deviceId: devices.audioInput.selected$.value?.id, echoCancellation, - noiseSuppression: noiseSuppression && !rnnoiseEnabled, + noiseSuppression: shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: noiseSuppression, + rnnoiseEnabled, + rnnoiseSupported: supportsRNNoiseProcessor(), + }), }); } From 1eec02a8e04f7ffabcbd36a0a04275da3ede0204 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 04:01:27 +0200 Subject: [PATCH 12/28] test(playwright): avoid hidden RNNoise input visibility assertions Signed-off-by: LucaPisl --- playwright/spa-call-sticky.spec.ts | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/playwright/spa-call-sticky.spec.ts b/playwright/spa-call-sticky.spec.ts index c97d02144..212bbee3a 100644 --- a/playwright/spa-call-sticky.spec.ts +++ b/playwright/spa-call-sticky.spec.ts @@ -15,6 +15,9 @@ import { import { SpaHelpers } from "./spa-helpers"; +const RNNOISE_LABEL = "Enable enhanced noise suppression (RNNoise)"; +const RNNOISE_TOGGLE_SELECTOR = "#activateRNNoiseSuppression"; + async function setupTwoUserSpaCall( browser: Browser, page: Page, @@ -172,18 +175,21 @@ async function interceptEventSend( ); } -async function enableRNNoiseInSettings(page: Page): Promise { +async function openAudioSettings(page: Page): Promise { await page.getByRole("button", { name: "Settings" }).click(); await page.getByRole("tab", { name: "Audio" }).click(); +} - const rnnoiseToggle = page.getByLabel( - "Enable enhanced noise suppression (RNNoise)", - ); - await expect(rnnoiseToggle).toBeVisible(); +async function enableRNNoiseInSettings(page: Page): Promise { + await openAudioSettings(page); + const rnnoiseLabel = page.locator("label", { hasText: RNNOISE_LABEL }); + await expect(rnnoiseLabel).toBeVisible(); + const rnnoiseToggle = page.locator(RNNOISE_TOGGLE_SELECTOR); const supported = await rnnoiseToggle.isEnabled(); if (supported && !(await rnnoiseToggle.isChecked())) { - await rnnoiseToggle.check(); + await rnnoiseLabel.click(); + await expect(rnnoiseToggle).toBeChecked(); } await page.getByTestId("modal_close").click(); @@ -191,12 +197,11 @@ async function enableRNNoiseInSettings(page: Page): Promise { } async function expectRNNoiseEnabledInSettings(page: Page): Promise { - await page.getByRole("button", { name: "Settings" }).click(); - await page.getByRole("tab", { name: "Audio" }).click(); + await openAudioSettings(page); - const rnnoiseToggle = page.getByLabel( - "Enable enhanced noise suppression (RNNoise)", - ); + const rnnoiseLabel = page.locator("label", { hasText: RNNOISE_LABEL }); + await expect(rnnoiseLabel).toBeVisible(); + const rnnoiseToggle = page.locator(RNNOISE_TOGGLE_SELECTOR); await expect(rnnoiseToggle).toBeChecked(); await page.getByTestId("modal_close").click(); From d41535e854695bc1aeec7174c1d44bcb9ae41975 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 04:01:54 +0200 Subject: [PATCH 13/28] test(playwright): add RNNoise device-switch stability coverage Signed-off-by: LucaPisl --- playwright.config.ts | 33 ++++++--- playwright/spa-call-sticky.spec.ts | 104 +++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 10 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index 4fb86b95b..d05fc50ae 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -11,6 +11,27 @@ import { defineConfig, devices } from "@playwright/test"; const baseURL = process.env.USE_DOCKER ? "http://localhost:8080" : "https://localhost:3000"; +const fakeAudioCaptureFile = process.env.PLAYWRIGHT_FAKE_AUDIO_CAPTURE_FILE; +const fakeVideoCaptureFile = process.env.PLAYWRIGHT_FAKE_VIDEO_CAPTURE_FILE; + +function buildFakeMediaArgs(): string[] { + const args = [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ]; + + if (fakeAudioCaptureFile) { + args.push(`--use-file-for-fake-audio-capture=${fakeAudioCaptureFile}`); + } + if (fakeVideoCaptureFile) { + args.push(`--use-file-for-fake-video-capture=${fakeVideoCaptureFile}`); + } + + return args; +} + +const fakeMediaArgs = buildFakeMediaArgs(); /** * See https://playwright.dev/docs/test-configuration. @@ -50,11 +71,7 @@ export default defineConfig({ ], ignoreHTTPSErrors: true, launchOptions: { - args: [ - "--use-fake-ui-for-media-stream", - "--use-fake-device-for-media-stream", - "--mute-audio", - ], + args: fakeMediaArgs, }, }, }, @@ -85,11 +102,7 @@ export default defineConfig({ "camera", ], launchOptions: { - args: [ - "--use-fake-ui-for-media-stream", - "--use-fake-device-for-media-stream", - "--mute-audio", - ], + args: fakeMediaArgs, }, }, }, diff --git a/playwright/spa-call-sticky.spec.ts b/playwright/spa-call-sticky.spec.ts index 212bbee3a..3e11840fd 100644 --- a/playwright/spa-call-sticky.spec.ts +++ b/playwright/spa-call-sticky.spec.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { expect, + type Locator, type Page, test, type Request, @@ -155,6 +156,81 @@ test("One to One rejoin after improper leave stays stable with RNNoise enabled", await expectRNNoiseEnabledInSettings(guestPage); }); +test("One to One call stays stable when switching devices with RNNoise enabled", async ({ + browser, + page, + browserName, +}) => { + const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); + + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); + test.skip( + !rnnoiseSupported, + "RNNoise is not supported in this browser environment", + ); + + await openAudioSettings(guestPage); + const microphoneDeviceRadios = await getDeviceSelectionRadios( + guestPage, + "Microphone", + ); + + // Some Chromium fake-device environments expose only one audio-input device, + // so device switching cannot be forced there. Fall back to output switching. + if (microphoneDeviceRadios.count < 2) { + const speakerDeviceRadios = await getDeviceSelectionRadios( + guestPage, + "Speaker", + ); + expect(speakerDeviceRadios.count).toBeGreaterThan(0); + + if (speakerDeviceRadios.count > 1) { + const selectedSpeakerBefore = await guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-output"), + ); + const targetSpeakerIndex = + speakerDeviceRadios.firstUncheckedIndex >= 0 + ? speakerDeviceRadios.firstUncheckedIndex + : 0; + await speakerDeviceRadios.radios.nth(targetSpeakerIndex).click(); + await expect + .poll(async () => + guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-output"), + ), + ) + .not.toBe(selectedSpeakerBefore); + } + } else { + const selectedMicrophoneBefore = await guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-input"), + ); + const targetMicrophoneIndex = + microphoneDeviceRadios.firstUncheckedIndex >= 0 + ? microphoneDeviceRadios.firstUncheckedIndex + : 1; + await microphoneDeviceRadios.radios.nth(targetMicrophoneIndex).click(); + await expect + .poll(async () => + guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-input"), + ), + ) + .not.toBe(selectedMicrophoneBefore); + } + + await guestPage.getByTestId("modal_close").click(); + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + await expect( + guestPage.getByRole("button", { name: "Mute microphone" }), + ).toBeVisible(); + await expectRNNoiseEnabledInSettings(guestPage); +}); + function isStickySend(url: string): boolean { return !!new URL(url).searchParams.get( "org.matrix.msc4354.sticky_duration_ms", @@ -180,6 +256,34 @@ async function openAudioSettings(page: Page): Promise { await page.getByRole("tab", { name: "Audio" }).click(); } +async function getDeviceSelectionRadios( + page: Page, + sectionHeading: string, +): Promise<{ + radios: Locator; + count: number; + firstUncheckedIndex: number; +}> { + const section = page + .locator("div") + .filter({ + has: page.getByRole("heading", { name: sectionHeading, exact: true }), + }) + .first(); + const radios = section.getByRole("radio"); + const count = await radios.count(); + const firstUncheckedIndex = await radios.evaluateAll((nodes) => + nodes.findIndex((node) => { + if (node instanceof HTMLInputElement) { + return !node.checked; + } + return node.getAttribute("aria-checked") !== "true"; + }), + ); + + return { radios, count, firstUncheckedIndex }; +} + async function enableRNNoiseInSettings(page: Page): Promise { await openAudioSettings(page); From 9140b5f00efe8467aa8f1b31e3d679b885f7f2fc Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 19:48:00 +0200 Subject: [PATCH 14/28] audio: make strong RNNoise suppression more aggressive Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 104 +++++++++++++++++++++++++++++ src/audio/RNNoiseProcessor.ts | 10 +-- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index d4c2dcb31..ed8faad04 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -84,6 +84,66 @@ function getGeneratedWorkletCode(): string { ); } +type WorkletPresetConfig = { + maxAttenuationDb: number; + openThreshold: number; + closeThreshold: number; + holdFrames: number; + attenuateMs: number; + releaseMs: number; +}; + +function getPresetConfig( + workletCode: string, + preset: "conservative" | "balanced" | "strong", +): WorkletPresetConfig { + const presetMatch = workletCode.match( + new RegExp(`${preset}:\\s*\\{([\\s\\S]*?)\\n\\s*\\},`), + ); + if (!presetMatch) { + throw new Error(`Could not find ${preset} preset in worklet code.`); + } + const presetBlock = presetMatch[1]; + const readNumber = (key: keyof WorkletPresetConfig): number => { + const keyMatch = presetBlock.match(new RegExp(`${key}:\\s*([0-9.]+)`)); + if (!keyMatch) { + throw new Error(`Could not find ${key} in ${preset} preset.`); + } + return Number(keyMatch[1]); + }; + + return { + maxAttenuationDb: readNumber("maxAttenuationDb"), + openThreshold: readNumber("openThreshold"), + closeThreshold: readNumber("closeThreshold"), + holdFrames: readNumber("holdFrames"), + attenuateMs: readNumber("attenuateMs"), + releaseMs: readNumber("releaseMs"), + }; +} + +function expectedAttenuationDb( + config: WorkletPresetConfig, + vadProbability: number, +): number { + if (vadProbability >= config.openThreshold) { + return 0; + } + + const thresholdRange = config.openThreshold - config.closeThreshold; + const attenuationProgress = + thresholdRange > 0 + ? Math.max( + 0, + Math.min( + 1, + (config.openThreshold - vadProbability) / thresholdRange, + ), + ) + : 1; + return attenuationProgress * config.maxAttenuationDb; +} + function instantiateWorkletProcessor(workletCode: string): { process: ( inputs: Float32Array[][], @@ -381,4 +441,48 @@ describe("RNNoiseProcessor", () => { expect(output[2]).toBeCloseTo(0.2, 6); expect(output).toHaveLength(first.length); }); + + it("keeps the balanced preset tuning unchanged", () => { + const balanced = getPresetConfig(getGeneratedWorkletCode(), "balanced"); + + expect(balanced).toEqual({ + maxAttenuationDb: 8, + openThreshold: 0.9, + closeThreshold: 0.55, + holdFrames: 10, + attenuateMs: 90, + releaseMs: 22, + }); + }); + + it("maps strong preset to a more aggressive profile than balanced", () => { + const workletCode = getGeneratedWorkletCode(); + const balanced = getPresetConfig(workletCode, "balanced"); + const strong = getPresetConfig(workletCode, "strong"); + + expect(strong.maxAttenuationDb).toBeGreaterThan(balanced.maxAttenuationDb); + expect(strong.openThreshold).toBeGreaterThanOrEqual(balanced.openThreshold); + expect(strong.closeThreshold).toBeGreaterThanOrEqual( + balanced.closeThreshold, + ); + expect(strong.holdFrames).toBeLessThan(balanced.holdFrames); + expect(strong.attenuateMs).toBeLessThan(balanced.attenuateMs); + }); + + it("applies lower expected noise-floor gain on strong than balanced", () => { + const workletCode = getGeneratedWorkletCode(); + const balanced = getPresetConfig(workletCode, "balanced"); + const strong = getPresetConfig(workletCode, "strong"); + const noiseLikeVadProbabilities = [0.2, 0.4, 0.6, 0.8]; + + for (const vad of noiseLikeVadProbabilities) { + expect(expectedAttenuationDb(strong, vad)).toBeGreaterThanOrEqual( + expectedAttenuationDb(balanced, vad), + ); + } + + const balancedSilenceGain = Math.pow(10, -balanced.maxAttenuationDb / 20); + const strongSilenceGain = Math.pow(10, -strong.maxAttenuationDb / 20); + expect(strongSilenceGain).toBeLessThan(balancedSilenceGain); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 66cbab58e..ecb5dea37 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -92,11 +92,11 @@ const PRESETS = { releaseMs: 22, }, strong: { - maxAttenuationDb: 12, - openThreshold: 0.88, - closeThreshold: 0.50, - holdFrames: 9, - attenuateMs: 70, + maxAttenuationDb: 16, + openThreshold: 0.90, + closeThreshold: 0.55, + holdFrames: 8, + attenuateMs: 55, releaseMs: 18, }, }; From 61c9cf3af71339f1daafa89f3207d0284218e1f7 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:11:13 +0200 Subject: [PATCH 15/28] test(audio): add regression for publisher policy after RNNoise toggle Signed-off-by: LucaPisl --- .../localMember/Publisher.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index fc7c41609..4c56be39e 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -489,6 +489,52 @@ describe("Publisher", () => { ); }); + it("restarts a newly-created publisher microphone track with native suppression disabled when RNNoise is already enabled", async () => { + await publisher.destroy(); + rnnoiseNoiseSuppression.setValue(true); + + const freshPublisher = new Publisher( + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + restartTrack: (...args: unknown[]) => void; + setProcessor: (...args: unknown[]) => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + try { + await flushPromises(); + + expect(micTrack.restartTrack).toHaveBeenCalledWith( + expect.objectContaining({ + noiseSuppression: false, + }), + ); + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + expect( + vi.mocked(micTrack.restartTrack).mock.invocationCallOrder[0], + ).toBeLessThan( + vi.mocked(micTrack.setProcessor).mock.invocationCallOrder[0], + ); + } finally { + await freshPublisher.destroy(); + } + }); + it("keeps native noise suppression enabled and skips processor when RNNoise is unsupported", async () => { vi.stubGlobal("AudioWorkletNode", undefined); vi.stubGlobal("MediaStreamAudioDestinationNode", undefined); From c0b150e95fd2ff10a164d62ce7bc387cc7fca4a7 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:11:46 +0200 Subject: [PATCH 16/28] fix(audio): sync native suppression policy on publisher creation Signed-off-by: LucaPisl --- .../CallViewModel/localMember/Publisher.ts | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index bea386c86..d4425c04e 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -63,6 +63,7 @@ export class Publisher { private readonly scope = new ObservableScope(); private rnnoiseOperationQueue: Promise = Promise.resolve(); + private rnnoisePolicySyncedTrack: LocalAudioTrack | null = null; /** * Creates a new Publisher. @@ -88,7 +89,7 @@ export class Publisher { // Setup track processor syncing (blur) this.observeTrackProcessors(this.scope, room, trackerProcessorState$); - this.observeRNNoiseProcessor(this.scope, room); + this.observeRNNoiseProcessor(this.scope, room, devices); this.observeRNNoiseSettingRestart(this.scope, room, devices); // Observe media device changes and update LiveKit active devices accordingly this.observeMediaDevices(this.scope, devices, controlledAudioDevices); @@ -437,6 +438,7 @@ export class Publisher { private observeRNNoiseProcessor( scope: ObservableScope, room: LivekitRoom, + devices: MediaDevices, ): void { const microphoneTrack$ = scope.behavior( observeTrackReference$( @@ -467,9 +469,24 @@ export class Publisher { ), ) .subscribe(([microphoneTrack, rnnoiseEnabled, rnnoisePreset]) => { - if (!microphoneTrack || !supportsRNNoiseProcessor()) return; + const rnnoiseSupported = supportsRNNoiseProcessor(); + if (!microphoneTrack || !rnnoiseSupported) { + this.rnnoisePolicySyncedTrack = microphoneTrack; + return; + } + + const isNewMicrophoneTrack = + microphoneTrack !== this.rnnoisePolicySyncedTrack; + this.rnnoisePolicySyncedTrack = microphoneTrack; this.enqueueRNNoiseOperation(async () => { + if (rnnoiseEnabled && isNewMicrophoneTrack) { + await this.restartMicrophoneTrackForNoiseSuppressionPolicy( + microphoneTrack, + devices, + rnnoiseEnabled, + ); + } await this.syncRNNoiseProcessor( microphoneTrack, rnnoiseEnabled, From 7e25641126341b6e2a6713b845613d733bc2de68 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:12:19 +0200 Subject: [PATCH 17/28] test(audio): assert RNNoiseProcessor stops processed track on destroy Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index ed8faad04..9ee19000c 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -26,14 +26,17 @@ type TestContext = { createDestinationNode: ReturnType; sourceNode: MediaStreamAudioSourceNode; destinationNode: MediaStreamAudioDestinationNode; - processedTrack: MediaStreamTrack; + processedTrack: MediaStreamTrack & { stop: ReturnType }; workletNode: AudioWorkletNode; audioContext: AudioContext; track: MediaStreamTrack; }; function createTestContext(sampleRate = 48000): TestContext { - const processedTrack = { id: "processed-track" } as MediaStreamTrack; + const processedTrack = { + id: "processed-track", + stop: vi.fn(), + } as unknown as MediaStreamTrack & { stop: ReturnType }; const sourceNode = { connect: vi.fn(), disconnect: vi.fn(), @@ -259,10 +262,17 @@ describe("RNNoiseProcessor", () => { expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ type: "destroy", }); + expect(t.processedTrack.stop).toHaveBeenCalledOnce(); expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:rnnoise"); expect(processor.processedTrack).toBeUndefined(); }); + it("destroy does not throw when no processed track exists", async () => { + const processor = new RNNoiseProcessor(); + + await expect(processor.destroy()).resolves.toBeUndefined(); + }); + it("restart re-initializes with a new processed track", async () => { const first = createTestContext(); const second = createTestContext(); From db9c568f1a72b6087f5ffed2fa427fb200b5eb53 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:12:41 +0200 Subject: [PATCH 18/28] fix(audio): stop processed track on destroy Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index ecb5dea37..2746dceda 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -431,6 +431,12 @@ export class RNNoiseProcessor implements TrackProcessor< this.workletNode?.disconnect(); this.destinationNode?.disconnect(); + try { + this.processedTrack?.stop(); + } catch (e) { + logger.warn("Failed to stop RNNoise processed track during destroy", e); + } + // Revoke the Blob URL if (this.blobUrl) { URL.revokeObjectURL(this.blobUrl); From f9d2e5aef914b38a9866eda83f63cc243bf6b80b Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:13:21 +0200 Subject: [PATCH 19/28] test(audio): cover stricter RNNoise support preflight and settings gating Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 13 +++++++++++++ src/settings/SettingsModal.test.tsx | 5 +++++ .../CallViewModel/localMember/Publisher.test.ts | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index 9ee19000c..1be972afc 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -331,6 +331,19 @@ describe("RNNoiseProcessor", () => { "MediaStreamAudioSourceNode", class MediaStreamAudioSourceNode {}, ); + vi.stubGlobal( + "AudioWorklet", + class AudioWorkletWithoutAddModule {}, + ); + expect(supportsRNNoiseProcessor()).toBe(false); + vi.stubGlobal( + "AudioWorklet", + class AudioWorklet { + public addModule(): Promise { + return Promise.resolve(); + } + }, + ); expect(supportsRNNoiseProcessor()).toBe(true); }); diff --git a/src/settings/SettingsModal.test.tsx b/src/settings/SettingsModal.test.tsx index b44d9cb3d..c4a93e437 100644 --- a/src/settings/SettingsModal.test.tsx +++ b/src/settings/SettingsModal.test.tsx @@ -165,6 +165,11 @@ describe("SettingsModal RNNoise controls", () => { "(Enhanced noise suppression is not supported by this browser.)", ), ).toBeInTheDocument(); + expect( + screen.queryByText( + "Pick a suppression profile. Stronger modes remove more keyboard noise but can sound more processed.", + ), + ).not.toBeInTheDocument(); }); it("persists RNNoise setting when toggled", async () => { diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 4c56be39e..26b07ac53 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -364,6 +364,14 @@ describe("Publisher", () => { describe("RNNoise", () => { beforeEach(() => { vi.stubGlobal("AudioWorkletNode", class AudioWorkletNode {}); + vi.stubGlobal( + "AudioWorklet", + class AudioWorklet { + public addModule(): Promise { + return Promise.resolve(); + } + }, + ); vi.stubGlobal( "MediaStreamAudioDestinationNode", class MediaStreamAudioDestinationNode {}, @@ -537,6 +545,7 @@ describe("Publisher", () => { it("keeps native noise suppression enabled and skips processor when RNNoise is unsupported", async () => { vi.stubGlobal("AudioWorkletNode", undefined); + vi.stubGlobal("AudioWorklet", undefined); vi.stubGlobal("MediaStreamAudioDestinationNode", undefined); vi.stubGlobal("MediaStreamAudioSourceNode", undefined); const micTrack = createMockLocalTrack( From 4519936c0ec4ebdd8f0508d1be9c8ed2f104223b Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:13:40 +0200 Subject: [PATCH 20/28] fix(audio): strengthen RNNoise support preflight checks Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 2746dceda..84b3ad4b7 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -24,6 +24,14 @@ const DEFAULT_RNNOISE_PRESET: RNNoiseSuppressionPreset = "conservative"; const loadedAudioWorklets = new WeakSet(); const warnedUnsupportedSampleRates = new Set(); +type RNNoiseSupportGlobal = typeof globalThis & { + AudioWorklet?: { + prototype?: { + addModule?: unknown; + }; + }; +}; + function createUnsupportedSampleRateError(sampleRate: number): Error { return new Error( `RNNoise requires an AudioContext sample rate of ${RNNOISE_REQUIRED_SAMPLE_RATE}Hz (received ${sampleRate}Hz).`, @@ -45,10 +53,14 @@ function warnUnsupportedSampleRate(sampleRate: number): void { * Whether the current runtime supports the required APIs for RNNoise. */ export function supportsRNNoiseProcessor(): boolean { + const workletPrototype = (globalThis as RNNoiseSupportGlobal).AudioWorklet + ?.prototype; + return ( typeof AudioWorkletNode !== "undefined" && typeof MediaStreamAudioDestinationNode !== "undefined" && - typeof MediaStreamAudioSourceNode !== "undefined" + typeof MediaStreamAudioSourceNode !== "undefined" && + typeof workletPrototype?.addModule === "function" ); } From 16550cbebdf9d23d94e343b9288da3bef41d026a Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:15:30 +0200 Subject: [PATCH 21/28] test(audio): make AudioWorklet stubs async for lint Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 4 ++-- src/state/CallViewModel/localMember/Publisher.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index 1be972afc..cb7a4bd56 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -339,8 +339,8 @@ describe("RNNoiseProcessor", () => { vi.stubGlobal( "AudioWorklet", class AudioWorklet { - public addModule(): Promise { - return Promise.resolve(); + public async addModule(): Promise { + return; } }, ); diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 26b07ac53..935d92df8 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -367,8 +367,8 @@ describe("Publisher", () => { vi.stubGlobal( "AudioWorklet", class AudioWorklet { - public addModule(): Promise { - return Promise.resolve(); + public async addModule(): Promise { + return; } }, ); From 9556fdf001ff87b57c88205e522af4de1071993c Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 21:16:47 +0200 Subject: [PATCH 22/28] test(audio): satisfy require-await in AudioWorklet stubs Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 2 +- src/state/CallViewModel/localMember/Publisher.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index cb7a4bd56..6e9ac35c5 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -340,7 +340,7 @@ describe("RNNoiseProcessor", () => { "AudioWorklet", class AudioWorklet { public async addModule(): Promise { - return; + await Promise.resolve(); } }, ); diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 935d92df8..b69fdc003 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -368,7 +368,7 @@ describe("Publisher", () => { "AudioWorklet", class AudioWorklet { public async addModule(): Promise { - return; + await Promise.resolve(); } }, ); From db68ce79f90312c6c4e50abf5cd0053d2c475c73 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 22:12:37 +0200 Subject: [PATCH 23/28] fix(audio): stabilize RNNoise worklet module loading Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.test.ts | 20 +- src/audio/RNNoiseProcessor.ts | 29 +-- src/audio/RNNoiseWorkletModule.ts | 303 +++++++++++++++++++++++++++++ src/types/jitsi-rnnoise-wasm.d.ts | 12 ++ 4 files changed, 324 insertions(+), 40 deletions(-) create mode 100644 src/audio/RNNoiseWorkletModule.ts create mode 100644 src/types/jitsi-rnnoise-wasm.d.ts diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index 6e9ac35c5..dad600b23 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -15,9 +15,10 @@ import { supportsRNNoiseProcessor, } from "./RNNoiseProcessor"; -vi.mock("@jitsi/rnnoise-wasm/dist/rnnoise-sync.js?raw", () => ({ - default: - "function createRNNWasmModuleSync(){}; export default createRNNWasmModuleSync;", +const RNNOISE_WORKLET_MODULE_URL = "/assets/RNNoiseWorkletModule.js"; + +vi.mock("./RNNoiseWorkletModule.ts?url", () => ({ + default: "/assets/RNNoiseWorkletModule.js", })); type TestContext = { @@ -202,13 +203,6 @@ function instantiateWorkletProcessor(workletCode: string): { describe("RNNoiseProcessor", () => { beforeEach(() => { - vi.stubGlobal( - "URL", - Object.assign(URL, { - createObjectURL: vi.fn().mockReturnValue("blob:rnnoise"), - revokeObjectURL: vi.fn(), - }), - ); vi.stubGlobal( "MediaStream", class MediaStream { @@ -233,7 +227,7 @@ describe("RNNoiseProcessor", () => { audioContext: t.audioContext, }); - expect(t.addModule).toHaveBeenCalledWith("blob:rnnoise"); + expect(t.addModule).toHaveBeenCalledWith(RNNOISE_WORKLET_MODULE_URL); expect(t.createSourceNode).toHaveBeenCalledOnce(); expect(t.createDestinationNode).toHaveBeenCalledOnce(); expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ @@ -263,7 +257,6 @@ describe("RNNoiseProcessor", () => { type: "destroy", }); expect(t.processedTrack.stop).toHaveBeenCalledOnce(); - expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:rnnoise"); expect(processor.processedTrack).toBeUndefined(); }); @@ -387,7 +380,7 @@ describe("RNNoiseProcessor", () => { expect(processor.processedTrack).toBeUndefined(); }); - it("releases the worklet blob URL when worklet registration fails", async () => { + it("propagates worklet registration failures", async () => { const t = createTestContext(); const workletCtor = vi.fn().mockReturnValue(t.workletNode); const addModuleError = new Error("Failed to register worklet module"); @@ -404,7 +397,6 @@ describe("RNNoiseProcessor", () => { ).rejects.toThrow(addModuleError); expect(workletCtor).not.toHaveBeenCalled(); - expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:rnnoise"); }); it("restarts with the last known audio context when restart omits audioContext", async () => { diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 84b3ad4b7..6558e820f 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -13,6 +13,7 @@ import type { TrackProcessor, } from "livekit-client"; import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; +import rnnoiseWorkletModuleUrl from "./RNNoiseWorkletModule.ts?url"; /** * The number of samples per frame expected by RNNoise (at 48kHz = 10ms). @@ -340,7 +341,6 @@ export class RNNoiseProcessor implements TrackProcessor< private sourceNode?: MediaStreamAudioSourceNode; private workletNode?: AudioWorkletNode; private destinationNode?: MediaStreamAudioDestinationNode; - private blobUrl?: string; private destroyed = false; private preset: RNNoiseSuppressionPreset; private lastAudioContext?: AudioContext; @@ -358,25 +358,8 @@ export class RNNoiseProcessor implements TrackProcessor< return; } - // Lazy-load the RNNoise sync WASM module code - const rnnoiseModule = - await import("@jitsi/rnnoise-wasm/dist/rnnoise-sync.js?raw"); - const workletCode = createWorkletCode( - rnnoiseModule.default as unknown as string, - ); - - // Create a Blob URL for the AudioWorklet module. - const blob = new Blob([workletCode], { type: "text/javascript" }); - const blobUrl = URL.createObjectURL(blob); - - try { - await audioContext.audioWorklet.addModule(blobUrl); - loadedAudioWorklets.add(audioContext); - this.blobUrl = blobUrl; - } catch (e) { - URL.revokeObjectURL(blobUrl); - throw e; - } + await audioContext.audioWorklet.addModule(rnnoiseWorkletModuleUrl); + loadedAudioWorklets.add(audioContext); } public async init(opts: AudioProcessorOptions): Promise { @@ -449,12 +432,6 @@ export class RNNoiseProcessor implements TrackProcessor< logger.warn("Failed to stop RNNoise processed track during destroy", e); } - // Revoke the Blob URL - if (this.blobUrl) { - URL.revokeObjectURL(this.blobUrl); - this.blobUrl = undefined; - } - this.sourceNode = undefined; this.workletNode = undefined; this.destinationNode = undefined; diff --git a/src/audio/RNNoiseWorkletModule.ts b/src/audio/RNNoiseWorkletModule.ts new file mode 100644 index 000000000..b551d7aa5 --- /dev/null +++ b/src/audio/RNNoiseWorkletModule.ts @@ -0,0 +1,303 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import createRNNWasmModuleSync from "@jitsi/rnnoise-wasm/dist/rnnoise-sync.js"; + +import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; + +declare abstract class AudioWorkletProcessor { + protected constructor(options?: AudioWorkletNodeOptions); + public readonly port: MessagePort; +} + +declare function registerProcessor( + name: string, + processorCtor: new (options?: AudioWorkletNodeOptions) => AudioWorkletProcessor, +): void; + +const FRAME_SIZE = 480; +const RING_SIZE = FRAME_SIZE * 3; +const SAMPLE_RATE = 48000; +const RNNOISE_WORKLET_NAME = "rnnoise-processor"; +const DEFAULT_PRESET: RNNoiseSuppressionPreset = "conservative"; + +type PresetConfig = { + maxAttenuationDb: number; + openThreshold: number; + closeThreshold: number; + holdFrames: number; + attenuateMs: number; + releaseMs: number; +}; + +type RNNoiseModule = { + HEAPF32: Float32Array; + [key: string]: unknown; +}; + +type WorkletMessage = + | { type: "destroy" } + | { type: "preset"; preset: RNNoiseSuppressionPreset }; + +const PRESETS: Record = { + conservative: { + maxAttenuationDb: 4, + openThreshold: 0.92, + closeThreshold: 0.6, + holdFrames: 12, + attenuateMs: 120, + releaseMs: 25, + }, + balanced: { + maxAttenuationDb: 8, + openThreshold: 0.9, + closeThreshold: 0.55, + holdFrames: 10, + attenuateMs: 90, + releaseMs: 22, + }, + strong: { + maxAttenuationDb: 16, + openThreshold: 0.9, + closeThreshold: 0.55, + holdFrames: 8, + attenuateMs: 55, + releaseMs: 18, + }, +}; + +function isPreset( + preset: unknown, +): preset is keyof typeof PRESETS & RNNoiseSuppressionPreset { + return typeof preset === "string" && preset in PRESETS; +} + +class RNNoiseWorkletProcessor extends AudioWorkletProcessor { + private ready = false; + private destroyed = false; + private readonly inBuf = new Float32Array(RING_SIZE); + private readonly outBuf = new Float32Array(RING_SIZE); + private inW = 0; + private inR = 0; + private outW = 0; + private outR = 0; + private currentGain = 1; + private targetGain = 1; + private holdFrames = 0; + private maxAttenuationDb = PRESETS[DEFAULT_PRESET].maxAttenuationDb; + private openThreshold = PRESETS[DEFAULT_PRESET].openThreshold; + private closeThreshold = PRESETS[DEFAULT_PRESET].closeThreshold; + private holdFramesConfig = PRESETS[DEFAULT_PRESET].holdFrames; + private attenuateStep = 1; + private releaseStep = 1; + private module?: RNNoiseModule; + private pcmBuf?: number; + private state: number | null = null; + private heapF32?: Float32Array; + + public constructor() { + super(); + + this.setPreset(DEFAULT_PRESET); + this.initRNNoise(); + + this.port.onmessage = (event: MessageEvent): void => { + if (event.data.type === "destroy") { + this.cleanup(); + } else if ( + event.data.type === "preset" && + isPreset(event.data.preset) + ) { + this.setPreset(event.data.preset); + } + }; + } + + private smoothingStepFromMs(ms: number): number { + if (ms <= 0) return 1; + const tau = ms / 1000; + return 1 - Math.exp(-1 / (SAMPLE_RATE * tau)); + } + + private setPreset(preset: RNNoiseSuppressionPreset): void { + if (!isPreset(preset)) return; + + const config = PRESETS[preset]; + this.maxAttenuationDb = config.maxAttenuationDb; + this.openThreshold = config.openThreshold; + this.closeThreshold = config.closeThreshold; + this.holdFramesConfig = config.holdFrames; + this.attenuateStep = this.smoothingStepFromMs(config.attenuateMs); + this.releaseStep = this.smoothingStepFromMs(config.releaseMs); + } + + private updateTargetGain(vadProbability: number): void { + if (vadProbability >= this.openThreshold) { + this.holdFrames = this.holdFramesConfig; + this.targetGain = 1; + return; + } + + if (this.holdFrames > 0) { + this.holdFrames -= 1; + this.targetGain = 1; + return; + } + + const thresholdRange = this.openThreshold - this.closeThreshold; + const attenuationProgress = + thresholdRange > 0 + ? Math.max( + 0, + Math.min(1, (this.openThreshold - vadProbability) / thresholdRange), + ) + : 1; + + const attenuationDb = attenuationProgress * this.maxAttenuationDb; + this.targetGain = Math.pow(10, -attenuationDb / 20); + } + + private ringAvailable(w: number, r: number): number { + let available = w - r; + if (available < 0) available += RING_SIZE; + return available; + } + + private initRNNoise(): void { + try { + const module = createRNNWasmModuleSync() as unknown as RNNoiseModule; + const malloc = module["_malloc"] as (size: number) => number; + const rnnoiseInit = module["_rnnoise_init"] as () => void; + const rnnoiseCreate = module["_rnnoise_create"] as () => number; + const pcmBuf = malloc(FRAME_SIZE * 4); + rnnoiseInit(); + const state = rnnoiseCreate(); + + this.module = module; + this.pcmBuf = pcmBuf; + this.state = state; + this.heapF32 = module.HEAPF32; + this.ready = true; + } catch (error) { + this.port.postMessage({ type: "error", message: String(error) }); + } + } + + private cleanup(): void { + if ( + this.module && + this.state !== null && + this.pcmBuf !== undefined + ) { + const rnnoiseDestroy = this.module["_rnnoise_destroy"] as ( + state: number, + ) => void; + const free = this.module["_free"] as (ptr: number) => void; + rnnoiseDestroy(this.state); + free(this.pcmBuf); + this.state = null; + } + this.destroyed = true; + } + + private processRNNoiseFrame(): void { + if ( + !this.module || + this.state === null || + this.pcmBuf === undefined || + !this.heapF32 + ) { + return; + } + + const heapIdx = this.pcmBuf >> 2; + + for (let i = 0; i < FRAME_SIZE; i++) { + this.heapF32[heapIdx + i] = this.inBuf[(this.inR + i) % RING_SIZE] * 32768; + } + this.inR = (this.inR + FRAME_SIZE) % RING_SIZE; + + const rnnoiseProcessFrame = this.module["_rnnoise_process_frame"] as ( + state: number, + input: number, + output: number, + ) => number; + const vadProbability = rnnoiseProcessFrame( + this.state, + this.pcmBuf, + this.pcmBuf, + ); + this.updateTargetGain(vadProbability); + + for (let i = 0; i < FRAME_SIZE; i++) { + const smoothingStep = + this.targetGain < this.currentGain + ? this.attenuateStep + : this.releaseStep; + this.currentGain += (this.targetGain - this.currentGain) * smoothingStep; + this.outBuf[(this.outW + i) % RING_SIZE] = + (this.heapF32[heapIdx + i] / 32768) * this.currentGain; + } + this.outW = (this.outW + FRAME_SIZE) % RING_SIZE; + } + + private mixInputChannels( + inputChannels: Float32Array[], + sampleIndex: number, + channelCount: number, + ): number { + let mixed = 0; + for (let i = 0; i < channelCount; i++) { + const channel = inputChannels[i]; + mixed += channel ? (channel[sampleIndex] ?? 0) : 0; + } + return mixed / channelCount; + } + + public process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean { + if (this.destroyed) return false; + + const inputChannels = inputs[0]; + const output = outputs[0]?.[0]; + + if (!inputChannels?.length || !output) return true; + + const blockSize = output.length; + const channelCount = inputChannels.length; + + if (!this.ready) { + for (let i = 0; i < blockSize; i++) { + output[i] = this.mixInputChannels(inputChannels, i, channelCount); + } + return true; + } + + for (let i = 0; i < blockSize; i++) { + this.inBuf[this.inW] = this.mixInputChannels(inputChannels, i, channelCount); + this.inW = (this.inW + 1) % RING_SIZE; + } + + while (this.ringAvailable(this.inW, this.inR) >= FRAME_SIZE) { + this.processRNNoiseFrame(); + } + + const outAvailable = this.ringAvailable(this.outW, this.outR); + const toRead = Math.min(blockSize, outAvailable); + + for (let i = 0; i < toRead; i++) { + output[i] = this.outBuf[this.outR]; + this.outR = (this.outR + 1) % RING_SIZE; + } + for (let i = toRead; i < blockSize; i++) { + output[i] = 0; + } + + return true; + } +} + +registerProcessor(RNNOISE_WORKLET_NAME, RNNoiseWorkletProcessor); diff --git a/src/types/jitsi-rnnoise-wasm.d.ts b/src/types/jitsi-rnnoise-wasm.d.ts new file mode 100644 index 000000000..c7ef1b9d9 --- /dev/null +++ b/src/types/jitsi-rnnoise-wasm.d.ts @@ -0,0 +1,12 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +declare module "@jitsi/rnnoise-wasm/dist/rnnoise-sync.js" { + const createRNNWasmModuleSync: () => unknown; + + export default createRNNWasmModuleSync; +} From e7dd7516158adf28c900784b009d6e2d5d7f3506 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 22:12:45 +0200 Subject: [PATCH 24/28] test(playwright): validate RNNoise device-switch stability in automation Signed-off-by: LucaPisl --- playwright.config.ts | 11 +- playwright/spa-call-sticky.spec.ts | 207 ++++++++++++++++------------- playwright/spa-helpers.ts | 1 + 3 files changed, 126 insertions(+), 93 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index d05fc50ae..794d82bd7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -13,6 +13,8 @@ const baseURL = process.env.USE_DOCKER : "https://localhost:3000"; const fakeAudioCaptureFile = process.env.PLAYWRIGHT_FAKE_AUDIO_CAPTURE_FILE; const fakeVideoCaptureFile = process.env.PLAYWRIGHT_FAKE_VIDEO_CAPTURE_FILE; +const disableChromiumSandbox = + process.env.PLAYWRIGHT_DISABLE_CHROMIUM_SANDBOX === "1"; function buildFakeMediaArgs(): string[] { const args = [ @@ -32,6 +34,9 @@ function buildFakeMediaArgs(): string[] { } const fakeMediaArgs = buildFakeMediaArgs(); +const chromiumLaunchArgs = disableChromiumSandbox + ? [...fakeMediaArgs, "--no-sandbox", "--disable-setuid-sandbox"] + : fakeMediaArgs; /** * See https://playwright.dev/docs/test-configuration. @@ -70,8 +75,9 @@ export default defineConfig({ "camera", ], ignoreHTTPSErrors: true, + chromiumSandbox: !disableChromiumSandbox, launchOptions: { - args: fakeMediaArgs, + args: chromiumLaunchArgs, }, }, }, @@ -101,8 +107,9 @@ export default defineConfig({ "microphone", "camera", ], + chromiumSandbox: !disableChromiumSandbox, launchOptions: { - args: fakeMediaArgs, + args: chromiumLaunchArgs, }, }, }, diff --git a/playwright/spa-call-sticky.spec.ts b/playwright/spa-call-sticky.spec.ts index 3e11840fd..d328770d9 100644 --- a/playwright/spa-call-sticky.spec.ts +++ b/playwright/spa-call-sticky.spec.ts @@ -12,6 +12,7 @@ import { test, type Request, type Browser, + type ConsoleMessage, } from "@playwright/test"; import { SpaHelpers } from "./spa-helpers"; @@ -23,6 +24,7 @@ async function setupTwoUserSpaCall( browser: Browser, page: Page, browserName: string, + callName = `HelloCall-${Date.now()}-${Math.floor(Math.random() * 10000)}`, ): Promise<{ guestPage: Page }> { test.skip( browserName === "firefox", @@ -43,7 +45,7 @@ async function setupTwoUserSpaCall( }, ); - await SpaHelpers.createCall(page, "Androl", "HelloCall", true, "2_0"); + await SpaHelpers.createCall(page, "Androl", callName, true, "2_0"); const inviteLink = await SpaHelpers.getCallInviteLink(page); @@ -74,8 +76,8 @@ async function setupTwoUserSpaCall( "2_0", ); // Assert both sides have sent sticky membership events - expect(androlHasSentStickyEvent).toEqual(true); - expect(pevaraHasSentStickyEvent).toEqual(true); + await expect.poll(() => androlHasSentStickyEvent).toBe(true); + await expect.poll(() => pevaraHasSentStickyEvent).toBe(true); return { guestPage }; } @@ -118,117 +120,140 @@ test("One to One rejoin after improper leave does not crash EC", async ({ await expect(guestPage.getByTestId("videoTile")).toHaveCount(2); }); -test("One to One rejoin after improper leave stays stable with RNNoise enabled", async ({ - browser, - page, - browserName, -}) => { - const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); +test.describe("RNNoise scenarios", () => { + test.describe.configure({ mode: "serial" }); - await SpaHelpers.expectVideoTilesCount(page, 2); - await SpaHelpers.expectVideoTilesCount(guestPage, 2); - - const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); test.skip( - !rnnoiseSupported, - "RNNoise is not supported in this browser environment", + ({ browserName }) => browserName !== "chromium", + "RNNoise scenarios are validated on Chromium fake-media infrastructure.", ); - await expect - .poll(async () => - guestPage.evaluate(() => - localStorage.getItem("matrix-setting-rnnoise-noise-suppression"), - ), - ) - .toBe("true"); + test("One to One rejoin after improper leave stays stable with RNNoise enabled", async ({ + browser, + page, + browserName, + }) => { + const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); - await guestPage.reload(); - await expect(guestPage.getByTestId("lobby_joinCall")).toBeVisible(); - await guestPage.getByTestId("lobby_joinCall").click(); + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); - // Rejoin after abrupt disconnect should remain stable with RNNoise enabled. - await expect(page.getByTestId("videoTile")).toHaveCount(3); - await expect(guestPage.getByTestId("videoTile")).toHaveCount(2); - await expect( - guestPage.getByRole("button", { name: "Mute microphone" }), - ).toBeVisible(); + const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); + test.skip( + !rnnoiseSupported, + "RNNoise is not supported in this browser environment", + ); - await expectRNNoiseEnabledInSettings(guestPage); -}); + await expect + .poll(async () => + guestPage.evaluate(() => + localStorage.getItem("matrix-setting-rnnoise-noise-suppression"), + ), + ) + .toBe("true"); -test("One to One call stays stable when switching devices with RNNoise enabled", async ({ - browser, - page, - browserName, -}) => { - const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); + await guestPage.reload(); + await expect(guestPage.getByTestId("lobby_joinCall")).toBeVisible(); + await guestPage.getByTestId("lobby_joinCall").click(); - await SpaHelpers.expectVideoTilesCount(page, 2); - await SpaHelpers.expectVideoTilesCount(guestPage, 2); + // Rejoin after abrupt disconnect should remain stable with RNNoise enabled. + await expect(page.getByTestId("videoTile")).toHaveCount(3); + await expect(guestPage.getByTestId("videoTile")).toHaveCount(2); + await expect( + guestPage.getByRole("button", { name: "Mute microphone" }), + ).toBeVisible(); - const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); - test.skip( - !rnnoiseSupported, - "RNNoise is not supported in this browser environment", - ); + await expectRNNoiseEnabledInSettings(guestPage); + }); - await openAudioSettings(guestPage); - const microphoneDeviceRadios = await getDeviceSelectionRadios( - guestPage, - "Microphone", - ); + test("One to One call stays stable when switching devices with RNNoise enabled", async ({ + browser, + page, + browserName, + }) => { + const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName); + + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + const rnnoiseSupported = await enableRNNoiseInSettings(guestPage); + test.skip( + !rnnoiseSupported, + "RNNoise is not supported in this browser environment", + ); + + const rnnoiseErrors: string[] = []; + const consoleHandler = (message: ConsoleMessage): void => { + if ( + message.type() === "error" && + /rnnoise|audio\s*worklet/i.test(message.text()) + ) { + rnnoiseErrors.push(message.text()); + } + }; + guestPage.on("console", consoleHandler); - // Some Chromium fake-device environments expose only one audio-input device, - // so device switching cannot be forced there. Fall back to output switching. - if (microphoneDeviceRadios.count < 2) { - const speakerDeviceRadios = await getDeviceSelectionRadios( + await openAudioSettings(guestPage); + const microphoneDeviceRadios = await getDeviceSelectionRadios( guestPage, - "Speaker", + "Microphone", ); - expect(speakerDeviceRadios.count).toBeGreaterThan(0); - if (speakerDeviceRadios.count > 1) { - const selectedSpeakerBefore = await guestPage.evaluate(() => - localStorage.getItem("matrix-setting-audio-output"), + // Some Chromium fake-device environments expose only one audio-input device, + // so device switching cannot be forced there. Fall back to output switching. + if (microphoneDeviceRadios.count < 2) { + const speakerDeviceRadios = await getDeviceSelectionRadios( + guestPage, + "Speaker", ); - const targetSpeakerIndex = - speakerDeviceRadios.firstUncheckedIndex >= 0 - ? speakerDeviceRadios.firstUncheckedIndex - : 0; - await speakerDeviceRadios.radios.nth(targetSpeakerIndex).click(); + expect(speakerDeviceRadios.count).toBeGreaterThan(0); + + if (speakerDeviceRadios.count > 1) { + const selectedSpeakerBefore = await guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-output"), + ); + const targetSpeakerIndex = + speakerDeviceRadios.firstUncheckedIndex >= 0 + ? speakerDeviceRadios.firstUncheckedIndex + : 0; + await speakerDeviceRadios.radios.nth(targetSpeakerIndex).click(); + await expect + .poll(async () => + guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-output"), + ), + ) + .not.toBe(selectedSpeakerBefore); + } + } else { + const selectedMicrophoneBefore = await guestPage.evaluate(() => + localStorage.getItem("matrix-setting-audio-input"), + ); + const targetMicrophoneIndex = + microphoneDeviceRadios.firstUncheckedIndex >= 0 + ? microphoneDeviceRadios.firstUncheckedIndex + : 1; + await microphoneDeviceRadios.radios.nth(targetMicrophoneIndex).click(); await expect .poll(async () => guestPage.evaluate(() => - localStorage.getItem("matrix-setting-audio-output"), + localStorage.getItem("matrix-setting-audio-input"), ), ) - .not.toBe(selectedSpeakerBefore); + .not.toBe(selectedMicrophoneBefore); } - } else { - const selectedMicrophoneBefore = await guestPage.evaluate(() => - localStorage.getItem("matrix-setting-audio-input"), - ); - const targetMicrophoneIndex = - microphoneDeviceRadios.firstUncheckedIndex >= 0 - ? microphoneDeviceRadios.firstUncheckedIndex - : 1; - await microphoneDeviceRadios.radios.nth(targetMicrophoneIndex).click(); - await expect - .poll(async () => - guestPage.evaluate(() => - localStorage.getItem("matrix-setting-audio-input"), - ), - ) - .not.toBe(selectedMicrophoneBefore); - } - await guestPage.getByTestId("modal_close").click(); - await SpaHelpers.expectVideoTilesCount(page, 2); - await SpaHelpers.expectVideoTilesCount(guestPage, 2); - await expect( - guestPage.getByRole("button", { name: "Mute microphone" }), - ).toBeVisible(); - await expectRNNoiseEnabledInSettings(guestPage); + await guestPage.getByTestId("modal_close").click(); + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + await expect( + guestPage.getByRole("button", { name: "Mute microphone" }), + ).toBeVisible(); + await expectRNNoiseEnabledInSettings(guestPage); + expect(rnnoiseErrors).toEqual([]); + + guestPage.off("console", consoleHandler); + }); }); function isStickySend(url: string): boolean { diff --git a/playwright/spa-helpers.ts b/playwright/spa-helpers.ts index 648f6e6e4..7e91e2524 100644 --- a/playwright/spa-helpers.ts +++ b/playwright/spa-helpers.ts @@ -95,6 +95,7 @@ async function setRtcModeFromSettings( page: Page, mode: RtcMode, ): Promise { + await expect(page.getByRole("button", { name: "Settings" })).toBeVisible(); await page.getByRole("button", { name: "Settings" }).click(); await page.getByRole("tab", { name: "Preferences" }).click(); await page.getByText("Developer mode", { exact: true }).check(); // Idempotent: won't uncheck if already checked From c81e63f7e4bd00d798633f74a8337116f1256815 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 23:18:42 +0200 Subject: [PATCH 25/28] chore: add THIRD_PARTY_NOTICES for rnnoise BSD attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rnnoise C library (xiph/rnnoise) is compiled to WebAssembly inside @jitsi/rnnoise-wasm and bundled verbatim into the Element Call web app bundle. The rnnoise BSD 3-Clause license requires that its copyright notice be reproduced in documentation or other materials provided with binary distributions. @jitsi/rnnoise-wasm does not carry a NOTICE file nor embed the upstream BSD notice in its generated JS/WASM artefacts, so Element Call — as the distributor of the binary — must supply the attribution itself. Add THIRD_PARTY_NOTICES at the repository root containing: - xiph/rnnoise BSD 3-Clause notice (Mozilla, Jean-Marc Valin, Xiph.Org Foundation, Mark Borgerding) - @jitsi/rnnoise-wasm Apache 2.0 notice (ESTOS GmbH, BlueJimp SARL) Signed-off-by: LucaPisl --- THIRD_PARTY_NOTICES | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 THIRD_PARTY_NOTICES diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..957e20ffc --- /dev/null +++ b/THIRD_PARTY_NOTICES @@ -0,0 +1,61 @@ +This file reproduces copyright notices and license terms for third-party +software components that are compiled into Element Call's distributed bundle +and carry obligations to reproduce their notices in binary distributions. + +------------------------------------------------------------------------------- + +rnnoise — Recurrent neural network for audio noise reduction +Compiled to WebAssembly and bundled via @jitsi/rnnoise-wasm +https://github.com/xiph/rnnoise + +Copyright (c) 2017, Mozilla +Copyright (c) 2007-2017, Jean-Marc Valin +Copyright (c) 2005-2017, Xiph.Org Foundation +Copyright (c) 2003-2004, Mark Borgerding + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of the Xiph.Org Foundation nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------- + +@jitsi/rnnoise-wasm — WebAssembly build and JS wrapper for rnnoise +https://github.com/jitsi/rnnoise-wasm + +Copyright (c) 2013 ESTOS GmbH +Copyright (c) 2013 BlueJimp SARL + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From c1f18d8a5ac26f06c3ae12507cdcb45bd12a51b2 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 23:18:53 +0200 Subject: [PATCH 26/28] fix(audio): address RNNoise implementation audit findings - Fix copyright header in RNNoiseProcessor.ts (was "2025 New Vector Ltd.", all other files in the feature use "2026 Element Creations Ltd.") - Pin @jitsi/rnnoise-wasm to exact version 0.2.1 (was "^0.2.1") to prevent unexpected upstream WASM changes being pulled automatically - Add double-init guard to RNNoiseProcessor.init(): tears down existing nodes before re-initialising so callers need not explicitly destroy first - Add post-await destroyed check in init() to abort cleanly if a concurrent destroy() ran during worklet registration - Add pendingWorkletRegistrations WeakMap mutex in ensureWorkletRegistered() to prevent concurrent addModule() calls on the same AudioContext - Add comment to createWorkletCode() warning that it is the test-harness copy of the worklet and must be kept in sync with RNNoiseWorkletModule.ts - Add three new unit tests covering: double-init node cleanup, concurrent ensureWorkletRegistered() deduplication, concurrent init+destroy safety Signed-off-by: LucaPisl --- package.json | 2 +- src/audio/RNNoiseProcessor.test.ts | 92 ++++++++++++++++++++++++++++++ src/audio/RNNoiseProcessor.ts | 42 ++++++++++---- 3 files changed, 124 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index d532c0e74..b7c873384 100644 --- a/package.json +++ b/package.json @@ -142,6 +142,6 @@ }, "packageManager": "yarn@4.7.0", "dependencies": { - "@jitsi/rnnoise-wasm": "^0.2.1" + "@jitsi/rnnoise-wasm": "0.2.1" } } diff --git a/src/audio/RNNoiseProcessor.test.ts b/src/audio/RNNoiseProcessor.test.ts index dad600b23..a026fac7c 100644 --- a/src/audio/RNNoiseProcessor.test.ts +++ b/src/audio/RNNoiseProcessor.test.ts @@ -500,4 +500,96 @@ describe("RNNoiseProcessor", () => { const strongSilenceGain = Math.pow(10, -strong.maxAttenuationDb / 20); expect(strongSilenceGain).toBeLessThan(balancedSilenceGain); }); + + it("init() called twice without destroy() cleans up previous nodes", async () => { + const first = createTestContext(); + const second = createTestContext(); + const workletCtor = vi + .fn() + .mockReturnValueOnce(first.workletNode) + .mockReturnValueOnce(second.workletNode); + vi.stubGlobal("AudioWorkletNode", workletCtor); + + const processor = new RNNoiseProcessor(); + await processor.init({ + kind: Track.Kind.Audio, + track: first.track, + audioContext: first.audioContext, + }); + await processor.init({ + kind: Track.Kind.Audio, + track: second.track, + audioContext: second.audioContext, + }); + + // First nodes must have been torn down + expect(first.sourceNode.disconnect).toHaveBeenCalledOnce(); + expect(first.workletNode.disconnect).toHaveBeenCalledOnce(); + expect(first.destinationNode.disconnect).toHaveBeenCalledOnce(); + expect(first.processedTrack.stop).toHaveBeenCalledOnce(); + + // Second nodes should now be active + expect(processor.processedTrack).toBe(second.processedTrack); + }); + + it("concurrent ensureWorkletRegistered calls only invoke addModule once", async () => { + const t = createTestContext(); + // Replace the default resolved mock with a controllable promise so both + // init() calls are in-flight at the same time. + let resolveAddModule!: () => void; + t.addModule.mockReturnValueOnce( + new Promise((resolve) => { + resolveAddModule = resolve; + }), + ); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + + const processor1 = new RNNoiseProcessor(); + const processor2 = new RNNoiseProcessor(); + + // Start both inits before either addModule resolves + const init1 = processor1.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + const init2 = processor2.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + resolveAddModule(); + await Promise.all([init1, init2]); + + expect(t.addModule).toHaveBeenCalledOnce(); + }); + + it("does not expose an active graph after concurrent init and destroy", async () => { + const t = createTestContext(); + let resolveAddModule!: () => void; + t.addModule.mockReturnValueOnce( + new Promise((resolve) => { + resolveAddModule = resolve; + }), + ); + vi.stubGlobal("AudioWorkletNode", vi.fn().mockReturnValue(t.workletNode)); + + const processor = new RNNoiseProcessor(); + const initPromise = processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + // destroy() races with the in-flight init(); let destroy complete first + await processor.destroy(); + + // Now let the worklet registration resolve and init() resume + resolveAddModule(); + await initPromise; + + // init() must have aborted after seeing destroyed=true; no track exposed + expect(processor.processedTrack).toBeUndefined(); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 6558e820f..de09f6b77 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. @@ -22,7 +22,9 @@ const RNNOISE_SAMPLE_LENGTH = 480; const RNNOISE_REQUIRED_SAMPLE_RATE = 48000; const RNNOISE_WORKLET_NAME = "rnnoise-processor"; const DEFAULT_RNNOISE_PRESET: RNNoiseSuppressionPreset = "conservative"; -const loadedAudioWorklets = new WeakSet(); +// Stores the addModule() promise per AudioContext: pending while in-flight, +// settled (resolved) once complete, absent on failure (cleared for retry). +const workletRegistrations = new WeakMap>(); const warnedUnsupportedSampleRates = new Set(); type RNNoiseSupportGlobal = typeof globalThis & { @@ -66,7 +68,15 @@ export function supportsRNNoiseProcessor(): boolean { } /** - * Generates the AudioWorklet processor code as a string. + * Generates the AudioWorklet processor code as a string, for use in tests. + * + * WARNING: This function is the **test harness** version of the worklet. + * The authoritative runtime implementation lives in `RNNoiseWorkletModule.ts`, + * which Vite compiles and loads as a separate script via the `?url` import. + * If the processor logic changes in `RNNoiseWorkletModule.ts` — frame size, + * ring buffer, preset constants, downmix algorithm, etc. — the generated + * code here **must be updated to match** or tests will diverge from runtime + * behaviour. * * The worklet loads the RNNoise WASM module synchronously (base64-inlined) * and processes audio in 480-sample frames. A ring buffer bridges the @@ -351,18 +361,25 @@ export class RNNoiseProcessor implements TrackProcessor< this.preset = preset; } - private async ensureWorkletRegistered( - audioContext: AudioContext, - ): Promise { - if (loadedAudioWorklets.has(audioContext)) { - return; - } + private ensureWorkletRegistered(audioContext: AudioContext): Promise { + const existing = workletRegistrations.get(audioContext); + if (existing) return existing; - await audioContext.audioWorklet.addModule(rnnoiseWorkletModuleUrl); - loadedAudioWorklets.add(audioContext); + const pending = audioContext.audioWorklet.addModule(rnnoiseWorkletModuleUrl); + workletRegistrations.set(audioContext, pending); + // On failure, remove the entry so the next call can retry. + pending.catch(() => { + workletRegistrations.delete(audioContext); + }); + return pending; } public async init(opts: AudioProcessorOptions): Promise { + // If already initialized, tear down previous nodes before re-initializing + // so callers don't need to explicitly call destroy() first. + if (this.workletNode !== undefined) { + await this.destroy(); + } this.destroyed = false; const { audioContext, track } = opts; @@ -373,6 +390,9 @@ export class RNNoiseProcessor implements TrackProcessor< await this.ensureWorkletRegistered(audioContext); + // A concurrent destroy() may have run while we awaited worklet registration. + if (this.destroyed) return; + // Build the audio processing graph: // MediaStreamSource → AudioWorkletNode (RNNoise) → MediaStreamDestination const sourceNode = audioContext.createMediaStreamSource( From 1b7877a385d4102f130ff06e3a87a4c082585389 Mon Sep 17 00:00:00 2001 From: LucaPisl Date: Tue, 3 Mar 2026 23:36:17 +0200 Subject: [PATCH 27/28] fix(audio): mark ensureWorkletRegistered as async for lint compliance The @typescript-eslint/promise-function-async rule requires functions that return Promise values to be declared async. While the runtime behaviour is equivalent, adding async keeps the linter clean and makes the intent explicit. Signed-off-by: LucaPisl --- src/audio/RNNoiseProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index de09f6b77..648267bd3 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -361,7 +361,7 @@ export class RNNoiseProcessor implements TrackProcessor< this.preset = preset; } - private ensureWorkletRegistered(audioContext: AudioContext): Promise { + private async ensureWorkletRegistered(audioContext: AudioContext): Promise { const existing = workletRegistrations.get(audioContext); if (existing) return existing; From 66feb02ccaa50b51618c976705bbb315e1bfaa94 Mon Sep 17 00:00:00 2001 From: melogale Date: Tue, 17 Mar 2026 03:22:38 -0400 Subject: [PATCH 28/28] fix --- src/audio/RNNoiseProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 1b62ef245..023758abd 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -13,7 +13,7 @@ import type { TrackProcessor, } from "livekit-client"; import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; -import rnnoiseWorkletModuleUrl from "./RNNoiseWorkletModule.ts?url"; +import rnnoiseWorkletModuleUrl from "./RNNoiseWorkletModule.ts?worker&url"; /** * The number of samples per frame expected by RNNoise (at 48kHz = 10ms).