diff --git a/.changeset/fix_voice_message_element_compat.md b/.changeset/fix_voice_message_element_compat.md new file mode 100644 index 000000000..57270a55a --- /dev/null +++ b/.changeset/fix_voice_message_element_compat.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +fix of compatibility of voice messages with element clients and style misshaps diff --git a/src/app/features/room/AudioMessageRecorder.tsx b/src/app/features/room/AudioMessageRecorder.tsx index f324d444c..073a8cb54 100644 --- a/src/app/features/room/AudioMessageRecorder.tsx +++ b/src/app/features/room/AudioMessageRecorder.tsx @@ -8,6 +8,7 @@ type AudioMessageRecorderProps = { onRequestClose: () => void; onWaveformUpdate: (waveform: number[]) => void; onAudioLengthUpdate: (length: number) => void; + onAudioCodecUpdate?: (codec: string) => void; }; // We use a react voice recorder library to handle the recording of audio messages, as it provides a simple API and handles the complexities of recording audio in the browser. @@ -19,6 +20,7 @@ export function AudioMessageRecorder({ onRequestClose, onWaveformUpdate, onAudioLengthUpdate, + onAudioCodecUpdate, }: AudioMessageRecorderProps) { const containerRef = useRef(null); const isDismissedRef = useRef(false); @@ -50,7 +52,7 @@ export function AudioMessageRecorder({ borderRadius: config.radii.R400, boxShadow: config.shadow.E200, padding: config.space.S400, - minWidth: 300, + width: 300, }} > Audio Message Recorder @@ -60,16 +62,20 @@ export function AudioMessageRecorder({ audioFile, waveform, audioLength, + audioCodec, }: { audioFile: Blob; waveform: number[]; audioLength: number; + audioCodec: string; }) => { if (isDismissedRef.current) return; // closes the recorder and sends the audio file back to the parent component to be uploaded and sent as a message onRecordingComplete(audioFile); onWaveformUpdate(waveform); onAudioLengthUpdate(audioLength); + // Pass the audio codec to the parent component + if (onAudioCodecUpdate) onAudioCodecUpdate(audioCodec); }} buttonBackgroundColor={color.SurfaceVariant.Container} buttonHoverBackgroundColor={color.SurfaceVariant.ContainerHover} diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index b280cf5d5..777e5d2f9 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -149,6 +149,7 @@ import { usePowerLevelsContext } from '$hooks/usePowerLevels'; import { useRoomCreators } from '$hooks/useRoomCreators'; import { useRoomPermissions } from '$hooks/useRoomPermissions'; import { AutocompleteNotice } from '$components/editor/autocomplete/AutocompleteNotice'; +import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec'; import { SchedulePickerDialog } from './schedule-send'; import * as css from './schedule-send/SchedulePickerDialog.css'; import { @@ -1096,7 +1097,7 @@ export const RoomInput = forwardRef( onRecordingComplete={(audioBlob) => { const file = new File( [audioBlob], - `sable-audio-message-${Date.now()}.ogg`, + `sable-audio-message-${Date.now()}.${getSupportedAudioExtension(audioBlob.type)}`, { type: audioBlob.type, } diff --git a/src/app/features/room/msgContent.ts b/src/app/features/room/msgContent.ts index 5536f7d05..e5de20f2e 100644 --- a/src/app/features/room/msgContent.ts +++ b/src/app/features/room/msgContent.ts @@ -155,28 +155,60 @@ export const getAudioMsgContent = ( audioLength?: number ): AudioMsgContent => { const { file, encInfo } = item; - const content: IContent = { + let content: IContent = { msgtype: MsgType.Audio, filename: file.name, - body: file.name, + body: item.body && item.body.length > 0 ? item.body : 'a voice message', format: 'org.matrix.custom.html', - formatted_body: file.name, + formatted_body: item.body && item.body.length > 0 ? item.body : 'a voice message', info: { mimetype: file.type, size: file.size, + duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, }, + + // Element-compatible unstable extensible-event keys 'org.matrix.msc1767.audio': { waveform: waveform?.map((v) => Math.round(v * 1024)), // scale waveform values to fit in 10 bits (0-1024) for more efficient storage, as per MSC1767 spec duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, // if marked as spoiler, set duration to 0 to hide it in clients that support msc1767 }, + 'org.matrix.msc1767.text': item.body && item.body.length > 0 ? item.body : 'a voice message', + 'org.matrix.msc3245.voice.v2': { + duration: !audioLength ? 0 : audioLength, + waveform: waveform?.map((v) => Math.round(v * 1024)), + }, + // for element compat + 'org.matrix.msc3245.voice': {}, }; if (encInfo) { content.file = { ...encInfo, url: mxc, }; + content = { + ...content, + + // Element-compatible unstable extensible-event keys + 'org.matrix.msc1767.file': { + name: file.name, + mimetype: file.type, + size: file.size, + file: content.file, + }, + }; } else { content.url = mxc; + content = { + ...content, + + // Element-compatible unstable extensible-event keys + 'org.matrix.msc1767.file': { + name: file.name, + mimetype: file.type, + size: file.size, + url: content.url, + }, + }; } if (item.body && item.body.length > 0) content.body = item.body; if (item.formatted_body && item.formatted_body.length > 0) { diff --git a/src/app/plugins/voice-recorder-kit/supportedCodec.ts b/src/app/plugins/voice-recorder-kit/supportedCodec.ts new file mode 100644 index 000000000..99481bde8 --- /dev/null +++ b/src/app/plugins/voice-recorder-kit/supportedCodec.ts @@ -0,0 +1,80 @@ +const safariPreferredCodecs = [ + // Safari works best with MP4/AAC but fails when strict codecs are defined on iOS. + // Prioritize the plain container to avoid NotSupportedError during MediaRecorder initialization. + 'audio/mp4', + 'audio/mp4;codecs=mp4a.40.2', + 'audio/mp4;codecs=mp4a.40.5', + 'audio/mp4;codecs=aac', + 'audio/aac', + // Fallbacks + 'audio/wav;codecs=1', + 'audio/wav', + 'audio/mpeg', +]; + +const defaultPreferredCodecs = [ + // Chromium / Firefox stable path. + 'audio/webm;codecs=opus', + 'audio/webm', + // Firefox + 'audio/ogg;codecs=opus', + 'audio/ogg;codecs=vorbis', + 'audio/ogg', + // Fallbacks + 'audio/wav;codecs=1', + 'audio/wav', + 'audio/mpeg', + // Keep MP4/AAC as late fallback for non-Safari browsers. + 'audio/mp4;codecs=mp4a.40.2', + 'audio/mp4;codecs=mp4a.40.5', + 'audio/mp4;codecs=aac', + 'audio/mp4', + 'audio/aac', + 'audio/ogg;codecs=speex', + 'audio/webm;codecs=vorbis', +]; + +/** + * Checks for supported audio codecs in the current browser and returns the first supported codec. + * If no supported codec is found, it returns null. + */ +export function getSupportedAudioCodec(): string | null { + if (!('MediaRecorder' in globalThis) || !globalThis.MediaRecorder) { + return null; + } + + const userAgent = globalThis.navigator?.userAgent ?? ''; + const isIOS = + /iPad|iPhone|iPod/.test(userAgent) || + // eslint-disable-next-line @typescript-eslint/no-deprecated + (globalThis.navigator?.platform === 'MacIntel' && globalThis.navigator?.maxTouchPoints > 1); + const isSafari = /^((?!chrome|android|crios|fxios|edgios).)*safari/i.test(userAgent) || isIOS; + + const preferredCodecs = isSafari ? safariPreferredCodecs : defaultPreferredCodecs; + const supportedCodec = preferredCodecs.find((codec) => MediaRecorder.isTypeSupported(codec)); + return supportedCodec || null; +} + +/** + * Returns the appropriate file extension for a given audio codec. + * This is used to ensure that the recorded audio file has the correct extension based on the codec used for recording. + */ +export function getSupportedAudioExtension(codec: string): string { + const baseType = codec.split(';')[0].trim(); + switch (baseType) { + case 'audio/ogg': + return 'ogg'; + case 'audio/webm': + return 'webm'; + case 'audio/mp4': + return 'm4a'; + case 'audio/mpeg': + return 'mp3'; + case 'audio/wav': + return 'wav'; + case 'audio/aac': + return 'aac'; + default: + return 'dat'; // default extension for unknown codecs + } +} diff --git a/src/app/plugins/voice-recorder-kit/types.ts b/src/app/plugins/voice-recorder-kit/types.ts index 42cccd271..9834303c2 100644 --- a/src/app/plugins/voice-recorder-kit/types.ts +++ b/src/app/plugins/voice-recorder-kit/types.ts @@ -7,6 +7,7 @@ export type VoiceRecorderStopPayload = { audioUrl: string; waveform: number[]; audioLength: number; + audioCodec: string; }; export type UseVoiceRecorderOptions = { diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index 454762463..a1eab6ddf 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -5,6 +5,7 @@ import type { RecorderState, VoiceRecorderStopPayload, } from './types'; +import { getSupportedAudioCodec, getSupportedAudioExtension } from './supportedCodec'; const BAR_COUNT = 40; const WAVEFORM_POINT_COUNT = 100; @@ -31,9 +32,21 @@ function downsampleWaveform(samples: number[], targetCount: number): number[] { return result; } +/** + * Custom React hook for recording voice messages using the MediaRecorder API. + * It manages the recording state, audio data, and provides functions to control the recording process (start, pause, stop, resume, play, etc.). + * It also handles audio visualization by analyzing the audio stream and generating levels for a visualizer. + * The hook supports multiple audio codecs and generates appropriate file extensions based on the supported codec. + */ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoiceRecorderReturn { const { autoStart = true, onStop, onDelete } = options; + /** + * The audio codec we will use + * we will choose depending on the browser support + */ + const audioCodec = getSupportedAudioCodec(); + const [isRecording, setIsRecording] = useState(false); const [isStopped, setIsStopped] = useState(false); const [isTemporaryStopped, setIsTemporaryStopped] = useState(false); @@ -67,9 +80,17 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic const isRestartingRef = useRef(false); const isTemporaryStopRef = useRef(false); const temporaryPreviewUrlRef = useRef(null); - // waveform samples collected during recording, used to generate waveform on stop. We collect all samples and downsample at the end to get a more accurate waveform, especially for short recordings. We use a ref to avoid causing re-renders on every sample. + /** + * waveform samples collected during recording, used to generate waveform on stop. + * We collect all samples and downsample at the end to get a more accurate waveform, especially for short recordings. + * We use a ref to avoid causing re-renders on every sample. + */ const waveformSamplesRef = useRef([]); - // Flag to indicate whether we should be collecting waveform samples. We need this because there can be a short delay between starting recording and the audio graph being set up, during which we might get some samples that we don't want to include in the waveform. + /** + * Flag to indicate whether we should be collecting waveform samples. + * We need this because there can be a short delay between starting recording + * and the audio graph being set up, during which we might get some samples that we don't want to include in the waveform. + */ const isCollectingWaveformRef = useRef(false); const cleanupStream = useCallback(() => { @@ -142,6 +163,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic audioUrl: url, waveform: waveformData, audioLength, + audioCodec: file.type, }; onStop(payload); }, @@ -187,7 +209,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic }, []); const setupAudioGraph = useCallback( - (stream: MediaStream) => { + (stream: MediaStream): MediaStream => { const audioContext = new AudioContext(); audioContextRef.current = audioContext; const source = audioContext.createMediaStreamSource(stream); @@ -198,9 +220,17 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic const dataArray = new Uint8Array(bufferLength); analyserRef.current = analyser; dataArrayRef.current = dataArray; + + // Fix for iOS Safari: routing the stream through a MediaStreamDestination + // prevents the AudioContext from "stealing" the track from the MediaRecorder + const destination = audioContext.createMediaStreamDestination(); source.connect(analyser); + analyser.connect(destination); + audioContext.resume().catch(() => {}); animateLevels(); + + return destination.stream; }, [animateLevels] ); @@ -237,15 +267,21 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const codec = getSupportedAudioCodec(); + if (!codec) { + setError('No supported audio codec found for recording.'); + cleanupStream(); + return; + } streamRef.current = stream; chunksRef.current = []; previousChunksRef.current = []; waveformSamplesRef.current = []; isCollectingWaveformRef.current = true; - setupAudioGraph(stream); + const recordedStream = setupAudioGraph(stream); startRecordingTimer(); - const mediaRecorder = new MediaRecorder(stream); + const mediaRecorder = new MediaRecorder(recordedStream, { mimeType: codec }); mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { @@ -276,9 +312,20 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic return; } - if (chunksRef.current.length === 0) return; + if (chunksRef.current.length === 0) { + if (isTemporaryStopRef.current) { + setIsTemporaryStopped(true); + setIsStopped(true); + isTemporaryStopRef.current = false; + } else { + setIsStopped(true); + setIsTemporaryStopped(false); + } + return; + } - const blob = new Blob(chunksRef.current, { type: 'audio/ogg' }); + const actualType = chunksRef.current[0]?.type || codec || 'audio/webm'; + const blob = new Blob(chunksRef.current, { type: actualType }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -286,7 +333,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic lastUrlRef.current = url; setAudioUrl(url); - const file = new File([blob], `voice-${Date.now()}.ogg`, { type: 'audio/ogg' }); + const file = new File( + [blob], + `voice-${Date.now()}.${getSupportedAudioExtension(actualType)}`, + { + type: actualType, + } + ); setAudioFile(file); const waveformData = downsampleWaveform(waveformSamplesRef.current, WAVEFORM_POINT_COUNT); @@ -303,7 +356,9 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic } }; - mediaRecorder.start(); + // Pass a timeslice to ensure Safari iOS periodically flushes chunks + // Otherwise Safari might fail to emit any chunks when stopped abruptly + mediaRecorder.start(1000); setIsRecording(true); setIsPaused(false); setIsStopped(false); @@ -363,10 +418,21 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isTemporaryStopRef.current = false; if (mediaRecorder.state === 'recording') { - mediaRecorder.requestData(); + try { + mediaRecorder.requestData(); + } catch { + // ignore + } + } + + try { + mediaRecorder.stop(); + } catch { + // ignore } - mediaRecorder.stop(); + // Let cleanupStream() be handled by mediaRecorder.onstop + // Calling it synchronously here can kill the stream before Safari finishes emitting data setIsStopped(true); setIsTemporaryStopped(false); setIsPaused(false); @@ -403,7 +469,23 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic if (mediaRecorder && mediaRecorder.state !== 'inactive') { previousChunksRef.current = [...chunksRef.current]; isTemporaryStopRef.current = false; - mediaRecorder.stop(); + + if (mediaRecorder.state === 'recording') { + try { + mediaRecorder.requestData(); + } catch { + // ignore + } + } + + try { + mediaRecorder.stop(); + } catch { + // ignore + } + + // Let cleanupStream() be handled by mediaRecorder.onstop + // Calling it synchronously here can kill the stream before Safari finishes emitting data setIsStopped(true); setIsTemporaryStopped(false); setIsPaused(false); @@ -446,7 +528,8 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic chunksRef.current.length > 0 ? chunksRef.current : previousChunksRef.current; if (allChunks.length > 0) { - const blob = new Blob(allChunks, { type: 'audio/ogg' }); + const actualType = allChunks[0]?.type || audioCodec || 'audio/webm'; + const blob = new Blob(allChunks, { type: actualType }); urlToPlay = URL.createObjectURL(blob); temporaryPreviewUrlRef.current = urlToPlay; } @@ -501,12 +584,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic } }, [ audioUrl, - cleanupAudioContext, - isPlaying, isPaused, + isPlaying, + audioCodec, + stopTimer, + cleanupAudioContext, setupPlaybackGraph, startPlaybackTimer, - stopTimer, ]); const handlePlay = useCallback(() => { @@ -563,13 +647,16 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); streamRef.current = stream; - setupAudioGraph(stream); + const recordedStream = setupAudioGraph(stream); // Force update seconds to the correct total time before starting timer setSeconds(pausedTimeRef.current); startRecordingTimer(); - const mediaRecorder = new MediaRecorder(stream); + const codec = getSupportedAudioCodec() || audioCodec; + const mediaRecorder = codec + ? new MediaRecorder(recordedStream, { mimeType: codec }) + : new MediaRecorder(recordedStream); mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { @@ -590,9 +677,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isCollectingWaveformRef.current = false; - if (chunksRef.current.length === 0) return; + if (chunksRef.current.length === 0) { + setIsStopped(true); + return; + } - const blob = new Blob(chunksRef.current, { type: 'audio/webm' }); + const actualType = chunksRef.current[0]?.type || audioCodec || 'audio/webm'; + const blob = new Blob(chunksRef.current, { type: actualType }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -600,7 +691,11 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic lastUrlRef.current = url; setAudioUrl(url); - const file = new File([blob], `voice-${Date.now()}.webm`, { type: 'audio/webm' }); + const file = new File( + [blob], + `voice-${Date.now()}.${getSupportedAudioExtension(blob.type)}`, + { type: blob.type } + ); setAudioFile(file); const waveformData = downsampleWaveform(waveformSamplesRef.current, WAVEFORM_POINT_COUNT); @@ -609,7 +704,9 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic emitStopPayload(file, url, waveformData, audioLength); }; - mediaRecorder.start(); + // Pass a timeslice to ensure Safari iOS periodically flushes chunks + // Otherwise Safari might fail to emit any chunks when stopped abruptly + mediaRecorder.start(1000); setIsRecording(true); setIsPaused(false); setIsStopped(false); @@ -639,6 +736,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isResumingRef.current = false; } }, [ + audioCodec, cleanupAudioContext, cleanupStream, emitStopPayload,