diff --git a/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h b/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h new file mode 100644 index 00000000..c9a2fd5b --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class dev_onvoid_webrtc_media_audio_CustomAudioSource */ + +#ifndef _Included_dev_onvoid_webrtc_media_audio_CustomAudioSource +#define _Included_dev_onvoid_webrtc_media_audio_CustomAudioSource +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_audio_CustomAudioSource + * Method: initialize + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_initialize + (JNIEnv *, jobject); + + /* + * Class: dev_onvoid_webrtc_media_audio_CustomAudioSource + * Method: pushAudio + * Signature: ([BIIII)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_pushAudio + (JNIEnv *, jobject, jbyteArray, jint, jint, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/include/media/SyncClock.h b/webrtc-jni/src/main/cpp/include/media/SyncClock.h new file mode 100644 index 00000000..283a9338 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/SyncClock.h @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_SYNC_CLOCK_H_ +#define JNI_WEBRTC_MEDIA_SYNC_CLOCK_H_ + +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/ntp_time.h" + +#include + +namespace jni +{ + // Synchronized Clock for A/V timing + class SyncClock + { + public: + SyncClock(); + + // Get current timestamp in microseconds + int64_t GetTimestampUs() const; + + // Get current timestamp in milliseconds + int64_t GetTimestampMs() const; + + // Get NTP timestamp for RTP synchronization + webrtc::NtpTime GetNtpTime() const; + + private: + std::chrono::steady_clock::time_point start_time_; + }; +} +#endif \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h b/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h new file mode 100644 index 00000000..56679726 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h @@ -0,0 +1,61 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_AUDIO_CUSTOM_AUDIO_SOURCE_H_ +#define JNI_WEBRTC_API_AUDIO_CUSTOM_AUDIO_SOURCE_H_ + +#include "api/media_stream_interface.h" +#include "rtc_base/ref_counted_object.h" + +#include "media/SyncClock.h" + +#include +#include +#include + +namespace jni +{ + class CustomAudioSource : public webrtc::AudioSourceInterface + { + public: + explicit CustomAudioSource(std::shared_ptr clock); + + // AudioSourceInterface implementation + void RegisterObserver(webrtc::ObserverInterface * observer) override; + void UnregisterObserver(webrtc::ObserverInterface * observer) override; + void AddSink(webrtc::AudioTrackSinkInterface * sink) override; + void RemoveSink(webrtc::AudioTrackSinkInterface * sink) override; + SourceState state() const override; + bool remote() const override; + + // Push audio data with synchronization + void PushAudioData(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames); + + // Set audio capture delay for synchronization adjustment + void SetAudioCaptureDelay(int64_t delay_us); + + private: + std::vector sinks_; + std::shared_ptr clock_; + //webrtc::CriticalSection crit_; + std::atomic total_samples_captured_; + int64_t audio_capture_delay_us_; + }; +} + +#endif \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h b/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h new file mode 100644 index 00000000..e6d923e7 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CUSTOM_VIDEO_SOURCE_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CUSTOM_VIDEO_SOURCE_H_ + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "media/base/adapted_video_track_source.h" +#include "rtc_base/ref_counted_object.h" + +#include "media/SyncClock.h" + +#include + +namespace jni +{ + class CustomVideoSource : public webrtc::AdaptedVideoTrackSource + { + public: + explicit CustomVideoSource(std::shared_ptr clock); + + // AdaptedVideoTrackSource implementation. + virtual bool is_screencast() const override; + virtual std::optional needs_denoising() const override; + SourceState state() const override; + bool remote() const override; + + void PushFrame(const webrtc::VideoFrame & frame); + + private: + std::shared_ptr clock_; + uint16_t frame_id_; + }; +} +#endif \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp new file mode 100644 index 00000000..a0dbca45 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_CustomAudioSource.h" +#include "JavaUtils.h" + +#include "media/audio/CustomAudioSource.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_initialize +(JNIEnv * env, jobject caller) +{ + std::shared_ptr sync_clock = std::make_shared(); + webrtc::scoped_refptr source = webrtc::make_ref_counted(sync_clock); + + SetHandle(env, caller, source.release()); +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_pushAudio +(JNIEnv * env, jobject caller, jbyteArray audioData, jint bits_per_sample, jint sampleRate, jint channels, jint frameCount) +{ + jni::CustomAudioSource * source = GetHandle(env, caller); + CHECK_HANDLE(source); + + jbyte * data = env->GetByteArrayElements(audioData, nullptr); + jsize length = env->GetArrayLength(audioData); + + if (data != nullptr) { + source->PushAudioData(data, bits_per_sample, sampleRate, channels, frameCount); + + env->ReleaseByteArrayElements(audioData, data, JNI_ABORT); + } +} \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/SyncClock.cpp b/webrtc-jni/src/main/cpp/src/media/SyncClock.cpp new file mode 100644 index 00000000..ecc1a915 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/SyncClock.cpp @@ -0,0 +1,43 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/SyncClock.h" + +namespace jni +{ + SyncClock::SyncClock() : + start_time_(std::chrono::steady_clock::now()) + { + } + + int64_t SyncClock::GetTimestampUs() const + { + auto now = std::chrono::steady_clock::now(); + auto duration = now - start_time_; + return std::chrono::duration_cast(duration).count(); + } + + int64_t SyncClock::GetTimestampMs() const + { + return GetTimestampUs() / 1000; + } + + webrtc::NtpTime SyncClock::GetNtpTime() const + { + int64_t ntp_time_ms = webrtc::Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); + return webrtc::NtpTime(ntp_time_ms); + } +} \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp b/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp new file mode 100644 index 00000000..0b90cbd6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp @@ -0,0 +1,82 @@ +#include "media/audio/CustomAudioSource.h" + +#include +#include + +namespace jni +{ + CustomAudioSource::CustomAudioSource(std::shared_ptr clock) : + clock_(clock), + total_samples_captured_(0), + audio_capture_delay_us_(0) + { + } + + void CustomAudioSource::RegisterObserver(webrtc::ObserverInterface * observer) + { + // Not implemented - not needed for custom sources + } + + void CustomAudioSource::UnregisterObserver(webrtc::ObserverInterface * observer) + { + // Not implemented - not needed for custom sources + } + + void CustomAudioSource::AddSink(webrtc::AudioTrackSinkInterface * sink) + { + //webrtc::CritScope lock(&crit_); + + sinks_.push_back(sink); + } + + void CustomAudioSource::RemoveSink(webrtc::AudioTrackSinkInterface * sink) + { + //webrtc::CritScope lock(&crit_); + + sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); + } + + webrtc::AudioSourceInterface::SourceState CustomAudioSource::state() const + { + return kLive; + } + + bool CustomAudioSource::remote() const + { + return false; + } + + void CustomAudioSource::PushAudioData(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames) + { + //webrtc::CritScope lock(&crit_); + + // Calculate absolute capture time + int64_t timestamp_us = clock_->GetTimestampUs(); + + // Apply delay if audio capture has inherent latency + timestamp_us -= audio_capture_delay_us_; + + // Calculate NTP time for this audio frame + int64_t ntp_time_ms = clock_->GetNtpTime().ToMs(); + + // Create absolute capture time + absl::optional absolute_capture_time_ms = timestamp_us / 1000; + + // Send to all sinks with timing information + for (auto * sink : sinks_) { + sink->OnData(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames, + absolute_capture_time_ms); + } + + // Update total samples for tracking + total_samples_captured_ += number_of_frames; + } + + void CustomAudioSource::SetAudioCaptureDelay(int64_t delay_us) + { + audio_capture_delay_us_ = delay_us; + } +} \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp new file mode 100644 index 00000000..db1f979a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/CustomVideoSource.h" + +namespace jni +{ + CustomVideoSource::CustomVideoSource(std::shared_ptr clock) : + clock_(clock), + frame_id_(0) + { + } + + bool CustomVideoSource::is_screencast() const + { + return false; + } + + std::optional CustomVideoSource::needs_denoising() const + { + return false; + } + + webrtc::MediaSourceInterface::SourceState CustomVideoSource::state() const + { + return kLive; + } + + bool CustomVideoSource::remote() const + { + return false; + } + + void CustomVideoSource::PushFrame(const webrtc::VideoFrame& frame) + { + // Create frame with proper timestamp + webrtc::VideoFrame timestamped_frame = frame; + + // Use synchronized clock for timestamp + int64_t timestamp_us = clock_->GetTimestampUs(); + timestamped_frame.set_timestamp_us(timestamp_us); + + // Set RTP timestamp (90kHz clock) + uint32_t rtp_timestamp = static_cast((timestamp_us * 90) / 1000); + timestamped_frame.set_rtp_timestamp(rtp_timestamp); + + // Set NTP time for synchronization + timestamped_frame.set_ntp_time_ms(clock_->GetNtpTime().ToMs()); + + // Increment frame ID + timestamped_frame.set_id(frame_id_++); + + OnFrame(timestamped_frame); + } +} \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index 6127c846..07c850de 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -98,19 +98,23 @@ namespace jni FireOnChanged(); } - bool VideoTrackDesktopSource::is_screencast() const { + bool VideoTrackDesktopSource::is_screencast() const + { return true; } - std::optional VideoTrackDesktopSource::needs_denoising() const { + std::optional VideoTrackDesktopSource::needs_denoising() const + { return false; } - webrtc::MediaSourceInterface::SourceState VideoTrackDesktopSource::state() const { + webrtc::MediaSourceInterface::SourceState VideoTrackDesktopSource::state() const + { return sourceState; } - bool VideoTrackDesktopSource::remote() const { + bool VideoTrackDesktopSource::remote() const + { return false; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java new file mode 100644 index 00000000..0461aa3d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +/** + * Custom implementation of an audio source for WebRTC that allows pushing audio data + * from external sources directly to the WebRTC audio pipeline. + * + * @author Alex Andres + */ +public class CustomAudioSource extends AudioTrackSource { + + /** + * Constructs a new CustomAudioSource instance. + */ + public CustomAudioSource() { + super(); + + initialize(); + } + + /** + * Pushes audio data to be processed by this audio source. + * + * @param audioData The raw audio data bytes to process. + * @param bits_per_sample The number of bits per sample (e.g., 8, 16, 32). + * @param sampleRate The sample rate of the audio in Hz (e.g., 44100, 48000). + * @param channels The number of audio channels (1 for mono, 2 for stereo). + * @param frameCount The number of frames in the provided audio data. + */ + public native void pushAudio(byte[] audioData, int bits_per_sample, + int sampleRate, int channels, int frameCount); + + /** + * Initializes the native resources required by this audio source. + */ + private native void initialize(); + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java new file mode 100644 index 00000000..a9e5db5e --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2025 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.onvoid.webrtc.TestBase; +import dev.onvoid.webrtc.media.MediaSource; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +class CustomAudioSourceTest extends TestBase { + + private CustomAudioSource customAudioSource; + + + @BeforeEach + void init() { + customAudioSource = new CustomAudioSource(); + } + + @AfterEach + void dispose() { + + } + + @Test + void stateAfterCreation() { + assertEquals(MediaSource.State.LIVE, customAudioSource.getState()); + } + + @Test + void addNullSink() { + assertThrows(NullPointerException.class, () -> { + AudioTrack audioTrack = factory.createAudioTrack("audioTrack", customAudioSource); + audioTrack.addSink(null); + audioTrack.dispose(); + }); + } + + @Test + void removeNullSink() { + assertThrows(NullPointerException.class, () -> { + AudioTrack audioTrack = factory.createAudioTrack("audioTrack", customAudioSource); + audioTrack.removeSink(null); + audioTrack.dispose(); + }); + } + + @Test + void addRemoveSink() { + AudioTrack audioTrack = factory.createAudioTrack("audioTrack", customAudioSource); + AudioTrackSink sink = (data, bitsPerSample, sampleRate, channels, frames) -> { }; + + audioTrack.addSink(sink); + audioTrack.removeSink(sink); + audioTrack.dispose(); + } + + @Test + void pushAudioData() { + // 16-bit, 48kHz, stereo, 10ms + testAudioFormat(16, 48000, 2, 480); + } + + @Test + void pushAudioWithDifferentFormats() { + testAudioFormat(8, 8000, 1, 80); // 8-bit, 8kHz, mono, 10ms + testAudioFormat(16, 16000, 1, 160); // 16-bit, 16kHz, mono, 10ms + testAudioFormat(16, 44100, 2, 441); // 16-bit, 44.1kHz, stereo, 10ms + testAudioFormat(16, 48000, 2, 480); // 16-bit, 48kHz, stereo, 10ms + } + + private void testAudioFormat(int bitsPerSample, int sampleRate, int channels, int frameCount) { + AudioTrack audioTrack = factory.createAudioTrack("audioTrack", customAudioSource); + + final AtomicBoolean dataReceived = new AtomicBoolean(false); + final AtomicInteger receivedBitsPerSample = new AtomicInteger(0); + final AtomicInteger receivedSampleRate = new AtomicInteger(0); + final AtomicInteger receivedChannels = new AtomicInteger(0); + final AtomicInteger receivedFrames = new AtomicInteger(0); + + AudioTrackSink testSink = (data, bits, rate, chans, frames) -> { + dataReceived.set(true); + receivedBitsPerSample.set(bits); + receivedSampleRate.set(rate); + receivedChannels.set(chans); + receivedFrames.set(frames); + }; + + audioTrack.addSink(testSink); + + // Create a buffer with test audio data (silence in this case). + int bytesPerSample = bitsPerSample / 8; + byte[] audioData = new byte[frameCount * channels * bytesPerSample]; + + customAudioSource.pushAudio(audioData, bitsPerSample, sampleRate, channels, frameCount); + + // Verify that our sink received the data with correct parameters. + assertTrue(dataReceived.get(), "Audio data was not received by the sink"); + assertEquals(bitsPerSample, receivedBitsPerSample.get(), "Bits per sample doesn't match"); + assertEquals(sampleRate, receivedSampleRate.get(), "Sample rate doesn't match"); + assertEquals(channels, receivedChannels.get(), "Channel count doesn't match"); + assertEquals(frameCount, receivedFrames.get(), "Frame count doesn't match"); + + // Clean up. + audioTrack.removeSink(testSink); + audioTrack.dispose(); + } +} \ No newline at end of file