From fe6f462015089701992a16638210578bcc09fd15 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 8 Nov 2026 14:24:37 +0100 Subject: [PATCH] feat: feed device-captured audio to senders through its track createAudioSource() now returns a native DeviceAudioSource. The audio device module's proxy runs the recorded audio through the factory's AudioProcessing and hands it to these sources' sinks, the senders among them, instead of to WebRTC's transport, which fed every send stream. Local sinks on a microphone track now receive the processed audio. --- docs/guide/audio/audio-processing.md | 2 +- .../cpp/include/JNI_ToneAudioDeviceModule.h | 21 ++ .../cpp/include/api/ProxyAudioDeviceModule.h | 28 +- .../cpp/include/api/ToneAudioDeviceModule.h | 68 +++++ .../include/media/audio/DeviceAudioSource.h | 128 +++++++++ .../cpp/src/JNI_PeerConnectionFactory.cpp | 42 ++- .../cpp/src/JNI_ToneAudioDeviceModule.cpp | 39 +++ .../cpp/src/api/ProxyAudioDeviceModule.cpp | 34 ++- .../cpp/src/api/ToneAudioDeviceModule.cpp | 140 ++++++++++ .../cpp/src/media/audio/DeviceAudioSource.cpp | 245 ++++++++++++++++ .../onvoid/webrtc/PeerConnectionFactory.java | 5 + .../onvoid/webrtc/client/LocalAudioImpl.java | 6 +- .../onvoid/webrtc/DeviceAudioSendTests.java | 264 ++++++++++++++++++ .../media/audio/ToneAudioDeviceModule.java | 40 +++ 14 files changed, 1036 insertions(+), 26 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/JNI_ToneAudioDeviceModule.h create mode 100644 webrtc-jni/src/main/cpp/include/api/ToneAudioDeviceModule.h create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/DeviceAudioSource.h create mode 100644 webrtc-jni/src/main/cpp/src/JNI_ToneAudioDeviceModule.cpp create mode 100644 webrtc-jni/src/main/cpp/src/api/ToneAudioDeviceModule.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/DeviceAudioSource.cpp create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/ToneAudioDeviceModule.java diff --git a/docs/guide/audio/audio-processing.md b/docs/guide/audio/audio-processing.md index 2fb204890..36f0edd08 100644 --- a/docs/guide/audio/audio-processing.md +++ b/docs/guide/audio/audio-processing.md @@ -300,7 +300,7 @@ audioProcessing.processStream( ## Speech in the Processed Microphone Audio -WebRTC passes the microphone's audio from the audio device module straight to the senders, so a sink on a microphone track gets nothing. The factory detects speech in that audio as it is sent, after echo cancellation, noise suppression and gain control, in native code, and tells a listener only when speech begins and ends: +A sink on a microphone track gets the audio its senders send, after echo cancellation, noise suppression and gain control, on WebRTC's capture thread. To learn only when speech begins and ends, without handling the audio in Java, let the factory detect speech in that audio in native code and tell a listener: ```java VoiceActivityListener listener = speaking -> { diff --git a/webrtc-jni/src/main/cpp/include/JNI_ToneAudioDeviceModule.h b/webrtc-jni/src/main/cpp/include/JNI_ToneAudioDeviceModule.h new file mode 100644 index 000000000..5fde6224d --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_ToneAudioDeviceModule.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule */ + +#ifndef _Included_dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule +#define _Included_dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule + * Method: initialize + * Signature: (I)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule_initialize + (JNIEnv *, jobject, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/ProxyAudioDeviceModule.h b/webrtc-jni/src/main/cpp/include/api/ProxyAudioDeviceModule.h index 856fef617..e1ca067fa 100644 --- a/webrtc-jni/src/main/cpp/include/api/ProxyAudioDeviceModule.h +++ b/webrtc-jni/src/main/cpp/include/api/ProxyAudioDeviceModule.h @@ -19,11 +19,15 @@ #include "api/audio/audio_device.h" #include "api/audio/audio_device_defines.h" +#include "api/audio/audio_processing.h" #include "api/scoped_refptr.h" #include "rtc_base/thread.h" +#include "media/audio/DeviceAudioSource.h" + #include #include +#include #include namespace jni @@ -41,6 +45,14 @@ namespace jni // AudioSendStream::SendAudioData and aborts the process, and interleaves // microphone frames with the pushed audio whenever it does not. // + // Recorded audio therefore never reaches WebRTC's own transport. While + // capture is enabled the proxy hands it to its DeviceAudioCapture, which + // runs it through the factory's AudioProcessing and on to the + // DeviceAudioSources, the sources PeerConnectionFactory.createAudioSource() + // creates. Their tracks' senders receive it through the tracks' sinks, like + // the senders of every other local source. Render calls still go to + // WebRTC, which feeds the same AudioProcessing the playout signal. + // // While capture is disabled, InitRecording() and StartRecording() fail so // AudioState never opens the device, and recorded data that still arrives // from the wrapped module (from a recording the application started through @@ -71,6 +83,14 @@ namespace jni void SetCaptureEnabled(bool enabled); bool CaptureEnabled() const; + // Sets the AudioProcessing the factory is built with, which the + // recorded audio runs through. Called before the factory exists. + void SetAudioProcessing(webrtc::scoped_refptr processing); + + // The capture the factory's DeviceAudioSources receive the recorded + // audio from. The sources share it, and may outlive this proxy. + std::shared_ptr DeviceCapture() const; + // webrtc::AudioDeviceModule implementation. int32_t ActiveAudioLayer(AudioLayer * audioLayer) const override; int32_t RegisterAudioCallback(webrtc::AudioTransport * audioCallback) override; @@ -154,11 +174,12 @@ namespace jni // The AudioTransport registered with the wrapped module in place of // the one WebRTC registers with this proxy. Render calls are always - // forwarded; recorded data is forwarded only while capture is enabled. + // forwarded to WebRTC; recorded data goes to the DeviceAudioCapture, + // and only while capture is enabled. class TransportGate : public webrtc::AudioTransport { public: - explicit TransportGate(const std::atomic & captureEnabled); + TransportGate(const std::atomic & captureEnabled, DeviceAudioCapture & capture); ~TransportGate() override = default; void SetTarget(webrtc::AudioTransport * target); @@ -203,11 +224,14 @@ namespace jni private: const std::atomic & captureEnabled_; + DeviceAudioCapture & capture_; std::atomic target_; }; webrtc::scoped_refptr delegate_; std::atomic captureEnabled_; + // Declared before the gate, which refers to it. + const std::shared_ptr deviceCapture_; TransportGate gate_; bool gateRegistered_; // The factory's worker thread, set while WebRTC's transport is registered. diff --git a/webrtc-jni/src/main/cpp/include/api/ToneAudioDeviceModule.h b/webrtc-jni/src/main/cpp/include/api/ToneAudioDeviceModule.h new file mode 100644 index 000000000..929d95612 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/ToneAudioDeviceModule.h @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_TONE_AUDIO_DEVICE_MODULE_H_ +#define JNI_WEBRTC_API_TONE_AUDIO_DEVICE_MODULE_H_ + +#include "api/HeadlessAudioDeviceModule.h" + +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" + +#include + +namespace jni +{ + // A HeadlessAudioDeviceModule that also records: while recording, it hands + // its transport a sine tone in real time, 10 ms at a time, the way a device + // module hands it a microphone. Only the tests reach it, so that they can + // send device-captured audio without audio hardware. + class ToneAudioDeviceModule : public HeadlessAudioDeviceModule + { + public: + static webrtc::scoped_refptr Create(int frequencyHz) + { + return webrtc::make_ref_counted(frequencyHz); + } + + explicit ToneAudioDeviceModule(int frequencyHz); + ~ToneAudioDeviceModule() override; + + int32_t RegisterAudioCallback(webrtc::AudioTransport * audioCallback) override; + int32_t Terminate() override; + int32_t StartRecording() override; + int32_t StopRecording() override; + + private: + static constexpr int kSampleRate = 48000; + static constexpr size_t kFramesPer10Ms = kSampleRate / 100; + + // Records 10 ms; returns false once recording stops. + bool RecordThreadProcess(); + + const int frequencyHz_; + + webrtc::Mutex recordMutex_; + bool recordRunning_ RTC_GUARDED_BY(recordMutex_) = false; + webrtc::AudioTransport * recordCallback_ RTC_GUARDED_BY(recordMutex_) = nullptr; + int64_t nextRecordMillis_ = 0; + uint64_t phase_ = 0; + + webrtc::PlatformThread recordThread_; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/DeviceAudioSource.h b/webrtc-jni/src/main/cpp/include/media/audio/DeviceAudioSource.h new file mode 100644 index 000000000..3d13b1fc5 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/DeviceAudioSource.h @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_AUDIO_DEVICE_AUDIO_SOURCE_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_DEVICE_AUDIO_SOURCE_H_ + +#include "api/audio/audio_frame.h" +#include "api/audio/audio_processing.h" +#include "api/audio_options.h" +#include "api/media_stream_interface.h" +#include "api/notifier.h" +#include "api/scoped_refptr.h" +#include "common_audio/resampler/include/push_resampler.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +#include +#include +#include +#include +#include + +namespace jni +{ + class DeviceAudioSource; + + // Hands the audio recorded by a factory's audio device module to the + // factory's DeviceAudioSources, after running it through the factory's + // AudioProcessing. + // + // WebRTC would otherwise process the recorded audio itself and feed it into + // every audio send stream of the factory, whatever the stream's track. Here + // it reaches only the senders of device-captured tracks, through the tracks' + // sinks, like the audio of every other kind of local source. The playout + // side is left to WebRTC, which feeds the same AudioProcessing the render + // signal, so echo cancellation sees the audio it has to remove. + // + // Shared by the factory's ProxyAudioDeviceModule, which feeds it on the + // recording thread, and by every DeviceAudioSource it created, which may + // outlive the module. + class DeviceAudioCapture + { + public: + DeviceAudioCapture() = default; + DeviceAudioCapture(const DeviceAudioCapture &) = delete; + DeviceAudioCapture & operator=(const DeviceAudioCapture &) = delete; + + // Sets the AudioProcessing the factory was built with. Called once, + // before the factory exists. + void SetAudioProcessing(webrtc::scoped_refptr processing); + + // Releases the AudioProcessing and every source, once the module is + // gone and no more audio can arrive. + void Detach(); + + // Called on the recording thread with 10 ms of interleaved 16-bit + // audio. Processes it and hands it to the sources that have sinks. + void OnRecordedData(const void * audioSamples, size_t frames, size_t bytesPerSample, + size_t channels, uint32_t sampleRate, uint32_t totalDelayMs, + bool keyPressed, std::optional estimatedCaptureTimeNs); + + // A source is registered while it has at least one sink, and held + // for that long, so it is alive whenever audio is handed to it. + void AddSource(webrtc::scoped_refptr source); + webrtc::scoped_refptr RemoveSource(DeviceAudioSource * source); + + private: + // Guards the processing state, which only the recording thread + // uses; it changes threads when the device is restarted. + webrtc::Mutex processMutex_; + webrtc::scoped_refptr processing_ RTC_GUARDED_BY(processMutex_); + webrtc::AudioFrame frame_ RTC_GUARDED_BY(processMutex_); + webrtc::PushResampler resampler_ RTC_GUARDED_BY(processMutex_); + + // Not held while the sources deliver: the list is copied first, so + // adding or removing a sink never waits for another source's sinks. + webrtc::Mutex sourcesMutex_; + std::vector> sources_ RTC_GUARDED_BY(sourcesMutex_); + }; + + // The source of a track that sends audio captured by the factory's audio + // device module, as PeerConnectionFactory.createAudioSource() creates it. + // Its sinks, the senders of its tracks among them, receive the recorded + // audio after the factory's AudioProcessing, on the recording thread. + class DeviceAudioSource : public webrtc::Notifier + { + public: + DeviceAudioSource(const webrtc::AudioOptions & options, std::shared_ptr capture); + ~DeviceAudioSource() override; + + // webrtc::MediaSourceInterface implementation. + SourceState state() const override; + bool remote() const override; + + // webrtc::AudioSourceInterface implementation. WebRTC applies the + // options to the factory's AudioProcessing when a sender starts. + const webrtc::AudioOptions options() const override; + void AddSink(webrtc::AudioTrackSinkInterface * sink) override; + void RemoveSink(webrtc::AudioTrackSinkInterface * sink) override; + + // Hands one processed frame to every sink. + void Deliver(const webrtc::AudioFrame & frame, std::optional absoluteCaptureTimeMs); + + private: + const webrtc::AudioOptions options_; + const std::shared_ptr capture_; + + // Held for a whole delivery, as CustomAudioSource does, so that a + // sink is not called once RemoveSink() has returned. + webrtc::Mutex mutex_; + std::vector sinks_ RTC_GUARDED_BY(mutex_); + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 0c7a9b65b..2c5d0d9ed 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -47,11 +47,13 @@ #include "api/ProxyAudioDeviceModule.h" #include "media/audio/CustomAudioSource.h" +#include "media/audio/DeviceAudioSource.h" #include "media/audio/codec/AudioCodecFactories.h" #include "media/video/codec/DefaultVideoCodecFactories.h" #include "media/video/codec/VideoDecoderFactoryWrapper.h" #include "media/video/codec/VideoEncoderFactoryWrapper.h" +#include "api/make_ref_counted.h" #include "api/media_stream_interface.h" #include "rtc_base/logging.h" #include "rtc_base/thread.h" @@ -159,7 +161,9 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize dependencies.signaling_thread = signalingThread.get(); dependencies.socket_factory = networkThread->socketserver(); dependencies.event_log_factory = std::make_unique(); - dependencies.env = webrtc::CreateEnvironment(std::move(fieldTrials)); + + webrtc::Environment environment = webrtc::CreateEnvironment(std::move(fieldTrials)); + dependencies.env = environment; dependencies.adm = proxy; dependencies.audio_encoder_factory = audioEncoderFactory; dependencies.audio_decoder_factory = audioDecoderFactory; @@ -167,6 +171,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize // application's module, or of the one built here. std::shared_ptr tap; + // The module is built here rather than by the factory, since the proxy + // runs the recorded audio through it and WebRTC the playout audio: both + // must reach the same module for echo cancellation to work. + webrtc::scoped_refptr audioProcessingModule; + if (processing != nullptr) { auto moduleTap = GetHandle>(env, audioProcessing, "captureTapHandle"); @@ -175,17 +184,24 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize } tap = *moduleTap; - dependencies.audio_processing_builder = webrtc::CustomAudioProcessing( - webrtc::scoped_refptr(processing)); + audioProcessingModule = webrtc::scoped_refptr(processing); } else { tap = std::make_shared(); - auto builder = std::make_unique(); - builder->SetCapturePostProcessing(jni::AudioCaptureTap::CreateProcessing(tap)); + webrtc::BuiltinAudioProcessingBuilder builder; + builder.SetCapturePostProcessing(jni::AudioCaptureTap::CreateProcessing(tap)); + + audioProcessingModule = builder.Build(environment); - dependencies.audio_processing_builder = std::move(builder); + if (!audioProcessingModule) { + throw jni::Exception("Create AudioProcessing failed"); + } } + + proxy->SetAudioProcessing(audioProcessingModule); + + dependencies.audio_processing_builder = webrtc::CustomAudioProcessing(audioProcessingModule); dependencies.video_encoder_factory = std::move(videoEncoderFactory); dependencies.video_decoder_factory = std::move(videoDecoderFactory); @@ -352,14 +368,16 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_createAud jni::NativeLease factory(env, caller); CHECK_HANDLEV(factory, nullptr); - auto audioOptions = jni::AudioOptions::toNative(env, jni::JavaLocalRef(env, jAudioOptions)); + jni::ProxyAudioDeviceModule * audioModule = GetHandle(env, caller, "audioModuleHandle"); + CHECK_HANDLEV(audioModule, nullptr); - webrtc::scoped_refptr audioSource = factory->CreateAudioSource(audioOptions); + auto audioOptions = jni::AudioOptions::toNative(env, jni::JavaLocalRef(env, jAudioOptions)); - if (audioSource == nullptr) { - env->Throw(jni::JavaError(env, "Create audio-track source failed")); - return nullptr; - } + // Not WebRTC's own LocalAudioSource, which leaves the recorded audio to + // reach the senders past the track: this source hands it to the track's + // sinks, its senders among them. See DeviceAudioCapture. + webrtc::scoped_refptr audioSource = + webrtc::make_ref_counted(audioOptions, audioModule->DeviceCapture()); return jni::JavaFactories::createOwning(env, std::move(audioSource)).release(); } diff --git a/webrtc-jni/src/main/cpp/src/JNI_ToneAudioDeviceModule.cpp b/webrtc-jni/src/main/cpp/src/JNI_ToneAudioDeviceModule.cpp new file mode 100644 index 000000000..9a08ec499 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_ToneAudioDeviceModule.cpp @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_ToneAudioDeviceModule.h" +#include "JavaBoundary.h" +#include "JavaError.h" +#include "NativeHandle.h" + +#include "api/ToneAudioDeviceModule.h" + +#include "api/scoped_refptr.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_ToneAudioDeviceModule_initialize +(JNIEnv * env, jobject caller, jint frequencyHz) +{ + jni::CallFromJava(env, [&]() { + webrtc::scoped_refptr audioModule = jni::ToneAudioDeviceModule::Create(frequencyHz); + + if (audioModule->Init() != 0) { + env->Throw(jni::JavaError(env, "Initialize ToneAudioDeviceModule failed")); + return; + } + + SetHandle(env, caller, audioModule.release()); + }); +} diff --git a/webrtc-jni/src/main/cpp/src/api/ProxyAudioDeviceModule.cpp b/webrtc-jni/src/main/cpp/src/api/ProxyAudioDeviceModule.cpp index c5f768d9f..364f63288 100644 --- a/webrtc-jni/src/main/cpp/src/api/ProxyAudioDeviceModule.cpp +++ b/webrtc-jni/src/main/cpp/src/api/ProxyAudioDeviceModule.cpp @@ -33,7 +33,8 @@ namespace jni ProxyAudioDeviceModule::ProxyAudioDeviceModule(webrtc::scoped_refptr delegate) : delegate_(std::move(delegate)), captureEnabled_(false), - gate_(captureEnabled_), + deviceCapture_(std::make_shared()), + gate_(captureEnabled_, *deviceCapture_), gateRegistered_(false), workerThread_(nullptr) { @@ -50,6 +51,9 @@ namespace jni RTC_LOG(LS_WARNING) << "ProxyAudioDeviceModule: the AudioDeviceModule kept its audio callback. " "Stop its playout and recording before disposing the PeerConnectionFactory."; } + + // No more audio is recorded for the sources, which may live on. + deviceCapture_->Detach(); } void ProxyAudioDeviceModule::SetCaptureEnabled(bool enabled) @@ -87,6 +91,16 @@ namespace jni return captureEnabled_.load(std::memory_order_acquire); } + void ProxyAudioDeviceModule::SetAudioProcessing(webrtc::scoped_refptr processing) + { + deviceCapture_->SetAudioProcessing(std::move(processing)); + } + + std::shared_ptr ProxyAudioDeviceModule::DeviceCapture() const + { + return deviceCapture_; + } + int32_t ProxyAudioDeviceModule::ActiveAudioLayer(AudioLayer * audioLayer) const { return delegate_->ActiveAudioLayer(audioLayer); @@ -421,8 +435,10 @@ namespace jni // TransportGate - ProxyAudioDeviceModule::TransportGate::TransportGate(const std::atomic & captureEnabled) : + ProxyAudioDeviceModule::TransportGate::TransportGate(const std::atomic & captureEnabled, + DeviceAudioCapture & capture) : captureEnabled_(captureEnabled), + capture_(capture), target_(nullptr) { } @@ -462,17 +478,19 @@ namespace jni { newMicLevel = currentMicLevel; - webrtc::AudioTransport * target = target_.load(std::memory_order_acquire); - - if (!target || !captureEnabled_.load(std::memory_order_acquire)) { + if (!captureEnabled_.load(std::memory_order_acquire)) { // Drop the frame. Send streams of this factory are fed by their // tracks' sources, not by the device. return 0; } - return target->RecordedDataIsAvailable(audioSamples, nSamples, nBytesPerSample, nChannels, - samplesPerSec, totalDelayMS, clockDrift, currentMicLevel, - keyPressed, newMicLevel, estimatedCaptureTimeNS); + // Never handed to WebRTC's transport, which would feed it into every + // send stream of the factory. The device sources' tracks take it to + // their senders instead. + capture_.OnRecordedData(audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec, + totalDelayMS, keyPressed, estimatedCaptureTimeNS); + + return 0; } int32_t ProxyAudioDeviceModule::TransportGate::NeedMorePlayData(size_t nSamples, diff --git a/webrtc-jni/src/main/cpp/src/api/ToneAudioDeviceModule.cpp b/webrtc-jni/src/main/cpp/src/api/ToneAudioDeviceModule.cpp new file mode 100644 index 000000000..bc053da8b --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/ToneAudioDeviceModule.cpp @@ -0,0 +1,140 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/ToneAudioDeviceModule.h" + +#include "rtc_base/time_utils.h" + +#include +#include +#include +#include + +namespace jni +{ + ToneAudioDeviceModule::ToneAudioDeviceModule(int frequencyHz) : + HeadlessAudioDeviceModule(kSampleRate, 1), + frequencyHz_(frequencyHz) + { + } + + ToneAudioDeviceModule::~ToneAudioDeviceModule() + { + // The base destructor stops only its own recording state. + ToneAudioDeviceModule::StopRecording(); + } + + int32_t ToneAudioDeviceModule::RegisterAudioCallback(webrtc::AudioTransport * audioCallback) + { + { + // Waits for a recording callback in progress, as the base class + // waits for a pull, so the old transport is not called afterwards. + webrtc::MutexLock lock(&recordMutex_); + + recordCallback_ = audioCallback; + } + + return HeadlessAudioDeviceModule::RegisterAudioCallback(audioCallback); + } + + int32_t ToneAudioDeviceModule::Terminate() + { + ToneAudioDeviceModule::StopRecording(); + + return HeadlessAudioDeviceModule::Terminate(); + } + + int32_t ToneAudioDeviceModule::StartRecording() + { + int32_t result = HeadlessAudioDeviceModule::StartRecording(); + + if (result != 0) { + return result; + } + + { + webrtc::MutexLock lock(&recordMutex_); + + if (recordRunning_) { + return 0; + } + + recordRunning_ = true; + } + + nextRecordMillis_ = webrtc::TimeMillis(); + recordThread_ = webrtc::PlatformThread::SpawnJoinable( + [this] { + while (RecordThreadProcess()) { + } + }, + "webrtc_tone_audio_module_record_thread", + webrtc::ThreadAttributes().SetPriority(webrtc::ThreadPriority::kRealtime)); + + return 0; + } + + int32_t ToneAudioDeviceModule::StopRecording() + { + { + webrtc::MutexLock lock(&recordMutex_); + + recordRunning_ = false; + } + + if (!recordThread_.empty()) { + recordThread_.Finalize(); + } + + return HeadlessAudioDeviceModule::StopRecording(); + } + + bool ToneAudioDeviceModule::RecordThreadProcess() + { + std::array samples; + + for (size_t i = 0; i < kFramesPer10Ms; ++i, ++phase_) { + const double angle = 2 * 3.14159265358979323846 * frequencyHz_ * static_cast(phase_) / kSampleRate; + + samples[i] = static_cast(8000 * std::sin(angle)); + } + + { + webrtc::MutexLock lock(&recordMutex_); + + if (!recordRunning_) { + return false; + } + + if (recordCallback_ != nullptr) { + uint32_t newMicLevel = 0; + + recordCallback_->RecordedDataIsAvailable(samples.data(), kFramesPer10Ms, sizeof(int16_t), 1, + kSampleRate, 0, 0, 0, false, newMicLevel, webrtc::TimeNanos()); + } + } + + nextRecordMillis_ += 10; + + const int64_t waitMillis = nextRecordMillis_ - webrtc::TimeMillis(); + + if (waitMillis > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(waitMillis)); + } + + return true; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/DeviceAudioSource.cpp b/webrtc-jni/src/main/cpp/src/media/audio/DeviceAudioSource.cpp new file mode 100644 index 000000000..908895e8f --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/DeviceAudioSource.cpp @@ -0,0 +1,245 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/DeviceAudioSource.h" + +#include "api/audio/audio_view.h" +#include "audio/remix_resample.h" +#include "modules/audio_processing/include/audio_frame_proxies.h" +#include "rtc_base/logging.h" + +#include +#include + +namespace jni +{ + namespace + { + // The highest rate the recorded audio is processed at, the rate of the + // Opus encoder. WebRTC chooses it from the encoders of the factory's + // send streams, which a source does not know; this is their usual + // maximum, and a sender resamples to its encoder's rate anyway. + constexpr int kMaxProcessingRateHz = 48000; + + // Sets the format the recorded audio is processed in: the lowest + // native rate of the AudioProcessing that keeps every frequency of the + // input, as AudioTransportImpl chooses it, and the input's channels. + void InitializeCaptureFrame(int inputRate, size_t inputChannels, webrtc::AudioFrame * frame) + { + const int minRate = std::min(inputRate, kMaxProcessingRateHz); + + for (int nativeRate : webrtc::AudioProcessing::kNativeSampleRatesHz) { + frame->SetSampleRateAndChannelSize(nativeRate); + + if (nativeRate >= minRate) { + break; + } + } + + frame->num_channels_ = inputChannels; + } + } + + // DeviceAudioCapture + + void DeviceAudioCapture::SetAudioProcessing(webrtc::scoped_refptr processing) + { + webrtc::MutexLock lock(&processMutex_); + + processing_ = std::move(processing); + } + + void DeviceAudioCapture::Detach() + { + std::vector> sources; + + { + webrtc::MutexLock lock(&processMutex_); + + processing_ = nullptr; + } + { + webrtc::MutexLock lock(&sourcesMutex_); + + sources.swap(sources_); + } + + // The sources are released here, outside the lock: one may be the last + // reference, and its destructor does not call back into this object. + } + + void DeviceAudioCapture::OnRecordedData(const void * audioSamples, size_t frames, size_t bytesPerSample, + size_t channels, uint32_t sampleRate, uint32_t totalDelayMs, + bool keyPressed, std::optional estimatedCaptureTimeNs) + { + // What AudioTransportImpl requires of recorded audio: 10 ms of 16-bit + // mono or stereo, which fits into one audio frame. + if (audioSamples == nullptr || channels < 1 || channels > 2 || bytesPerSample != 2 * channels || + sampleRate < 8000 || frames * 100 != sampleRate || + frames * channels > webrtc::AudioFrame::kMaxDataSizeSamples) { + RTC_LOG(LS_WARNING) << "DeviceAudioCapture: dropping recorded audio of an unsupported format: " + << frames << " frames, " << channels << " channels, " << sampleRate << " Hz"; + return; + } + + std::vector> sources; + + { + webrtc::MutexLock lock(&sourcesMutex_); + + sources = sources_; + } + + webrtc::MutexLock lock(&processMutex_); + + webrtc::InterleavedView source(static_cast(audioSamples), frames, channels); + + frame_.Reset(); + InitializeCaptureFrame(static_cast(sampleRate), channels, &frame_); + webrtc::voe::RemixAndResample(source, static_cast(sampleRate), &resampler_, &frame_); + + // Processed even without a source, as WebRTC does, so that the capture + // post-processing, the voice activity detection among it, keeps seeing + // a recording the application started itself. + if (processing_) { + processing_->set_stream_delay_ms(static_cast(totalDelayMs)); + processing_->set_stream_key_pressed(keyPressed); + + int error = webrtc::ProcessAudioFrame(processing_.get(), &frame_); + + if (error != webrtc::AudioProcessing::kNoError) { + RTC_LOG(LS_WARNING) << "DeviceAudioCapture: ProcessStream() failed: " << error; + } + } + + std::optional captureTimeMs; + + if (estimatedCaptureTimeNs) { + captureTimeMs = *estimatedCaptureTimeNs / 1000000; + } + + for (const auto & deviceSource : sources) { + deviceSource->Deliver(frame_, captureTimeMs); + } + } + + void DeviceAudioCapture::AddSource(webrtc::scoped_refptr source) + { + webrtc::MutexLock lock(&sourcesMutex_); + + sources_.push_back(std::move(source)); + } + + webrtc::scoped_refptr DeviceAudioCapture::RemoveSource(DeviceAudioSource * source) + { + webrtc::MutexLock lock(&sourcesMutex_); + + auto it = std::find_if(sources_.begin(), sources_.end(), [source](const auto & registered) { + return registered.get() == source; + }); + + if (it == sources_.end()) { + return nullptr; + } + + webrtc::scoped_refptr removed = std::move(*it); + sources_.erase(it); + + return removed; + } + + // DeviceAudioSource + + DeviceAudioSource::DeviceAudioSource(const webrtc::AudioOptions & options, + std::shared_ptr capture) : + options_(options), + capture_(std::move(capture)) + { + } + + DeviceAudioSource::~DeviceAudioSource() + { + // A source with sinks is held by the capture, so it is destroyed only + // once its last sink is gone and the capture has let it go. + } + + webrtc::MediaSourceInterface::SourceState DeviceAudioSource::state() const + { + return kLive; + } + + bool DeviceAudioSource::remote() const + { + return false; + } + + const webrtc::AudioOptions DeviceAudioSource::options() const + { + return options_; + } + + void DeviceAudioSource::AddSink(webrtc::AudioTrackSinkInterface * sink) + { + webrtc::MutexLock lock(&mutex_); + + if (sink == nullptr || std::find(sinks_.begin(), sinks_.end(), sink) != sinks_.end()) { + return; + } + + sinks_.push_back(sink); + + // Registered while this lock is held, so that registering and + // unregistering follow the order of the sinks. The capture never takes + // this lock while holding its own. + if (sinks_.size() == 1) { + capture_->AddSource(webrtc::scoped_refptr(this)); + } + } + + void DeviceAudioSource::RemoveSink(webrtc::AudioTrackSinkInterface * sink) + { + // The capture's reference is released after the lock: it is never the + // last one, since the caller holds the track, but should it be, this + // object must not be destroyed while its own lock is held. + webrtc::scoped_refptr released; + + { + webrtc::MutexLock lock(&mutex_); + + auto it = std::find(sinks_.begin(), sinks_.end(), sink); + + if (it == sinks_.end()) { + return; + } + + sinks_.erase(it); + + if (sinks_.empty()) { + released = capture_->RemoveSource(this); + } + } + } + + void DeviceAudioSource::Deliver(const webrtc::AudioFrame & frame, std::optional absoluteCaptureTimeMs) + { + webrtc::MutexLock lock(&mutex_); + + for (auto * sink : sinks_) { + sink->OnData(frame.data(), 16, frame.sample_rate_hz_, frame.num_channels_, + frame.samples_per_channel_, absoluteCaptureTimeMs); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index c546377ac..18e9ecb81 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -257,6 +257,11 @@ public static Builder builder() { * factory's audio device module. The audio source may be used by one or * more {@link AudioTrack}s. *

+ * The audio reaches the senders of these tracks through the tracks, after + * the factory's audio processing: echo cancellation, noise suppression and + * gain control as the options ask for. A sink added to such a track gets + * the same audio, on WebRTC's capture thread, while the device records. + *

* Calling this commits the factory to device-captured audio and lets it open * the recording device; see the class description. * diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java index a1a872afa..17dde651b 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java @@ -29,9 +29,9 @@ /** * A local audio track, from the microphone or pushed audio. Speech is * detected in native code. Pushed audio reaches the detector as a sink of the - * native track. A microphone's source hands no audio to sinks, as WebRTC - * passes captured audio straight to the senders, so a microphone track takes - * its speech from the runtime's audio processing instead. + * native track. A microphone track takes its speech from the detector of the + * runtime's audio processing instead, which sees the processed microphone + * audio once, whatever the number of microphone tracks. */ class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java new file mode 100644 index 000000000..33886cb7f --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java @@ -0,0 +1,264 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; +import dev.onvoid.webrtc.media.audio.ToneAudioDeviceModule; + +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; + +/** + * Audio captured by the factory's audio device module reaches the senders + * through the track of a device source, like the audio of every other local + * source. The module here records a tone in place of a microphone. + */ +class DeviceAudioSendTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + /** Well above the noise of a codec, well below the recorded tone. */ + private static final int AUDIBLE = 1000; + + + @Test + void localSinkReceivesRecording() throws Exception { + try (DeviceCall call = new DeviceCall(new AudioOptions())) { + CountDownLatch audible = new CountDownLatch(5); + AudioTrackSink sink = (data, bits, rate, channels, frames) -> { + if (isAudible(data, bits, channels, frames)) { + audible.countDown(); + } + }; + + // WebRTC's own source of device-captured audio never fed a local + // sink: the recording reached its senders past the track. + call.track.addSink(sink); + + try { + call.connect(); + + assertTrue(audible.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "No recorded audio at the local sink"); + } + finally { + call.track.removeSink(sink); + } + } + } + + @Test + void calleeReceivesRecording() throws Exception { + try (DeviceCall call = new DeviceCall(new AudioOptions())) { + CountDownLatch audible = new CountDownLatch(5); + + call.connect(); + + RTCRtpReceiver receiver = call.getReceiver(); + AudioTrack remote = (AudioTrack) receiver.getTrack(); + AudioTrackSink sink = (data, bits, rate, channels, frames) -> { + if (isAudible(data, bits, channels, frames)) { + audible.countDown(); + } + }; + + remote.addSink(sink); + + try { + assertTrue(audible.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "No recorded audio at the callee"); + } + finally { + remote.removeSink(sink); + receiver.dispose(); + } + } + } + + @Test + void mediaSourceStatsReportLevel() throws Exception { + try (DeviceCall call = new DeviceCall(new AudioOptions())) { + call.connect(); + + // The level is computed by the send stream from what it is handed, + // so it shows that the recording reaches the sender. + Map stats = call.awaitMediaSourceStats(attributes -> + number(attributes, "audioLevel") > 0 && number(attributes, "totalAudioEnergy") > 0); + + assertNotNull(stats, "The media source reports no audio level"); + } + } + + @Test + void echoStatisticsReported() throws Exception { + AudioOptions options = new AudioOptions(); + options.echoCancellation = true; + + try (DeviceCall call = new DeviceCall(options)) { + call.connect(); + + // The echo canceller reports once it has seen both sides: the + // recording, run through it by the audio device module's proxy, + // and the playout of the callee, run through it by WebRTC. + Map stats = call.awaitMediaSourceStats(attributes -> + attributes.containsKey("echoReturnLoss")); + + assertNotNull(stats, "The media source reports no echo statistics"); + } + } + + private static boolean isAudible(byte[] data, int bits, int channels, int frames) { + if (bits != 16) { + return false; + } + + ByteBuffer samples = ByteBuffer.wrap(data).order(ByteOrder.nativeOrder()); + int count = Math.min(frames * channels, data.length / 2); + + for (int i = 0; i < count; i++) { + if (Math.abs(samples.getShort(i * 2)) > AUDIBLE) { + return true; + } + } + + return false; + } + + private static double number(Map attributes, String name) { + Object value = attributes.get(name); + + return value instanceof Number ? ((Number) value).doubleValue() : 0; + } + + + + /** + * A call that sends the track of a device source from the caller to the + * callee, both on a factory whose audio device module records a tone. + */ + private static class DeviceCall implements AutoCloseable { + + final ToneAudioDeviceModule audioModule; + + final PeerConnectionFactory factory; + + final AudioTrackSource source; + + final AudioTrack track; + + final TestPeerConnection caller; + + final TestPeerConnection callee; + + final RTCRtpSender sender; + + + DeviceCall(AudioOptions options) { + audioModule = new ToneAudioDeviceModule(440); + factory = new PeerConnectionFactory(audioModule); + source = factory.createAudioSource(options); + track = factory.createAudioTrack("microphone", source); + + caller = new TestPeerConnection(factory); + callee = new TestPeerConnection(factory); + + caller.setRemotePeerConnection(callee); + callee.setRemotePeerConnection(caller); + + sender = caller.getPeerConnection().addTrack(track, Collections.singletonList("stream")); + } + + void connect() throws Exception { + callee.setRemoteDescription(caller.createOffer()); + caller.setRemoteDescription(callee.createAnswer()); + + caller.waitUntilConnected(); + callee.waitUntilConnected(); + } + + /** + * Returns the callee's receiver of the track, which is the caller's to + * dispose. + */ + RTCRtpReceiver getReceiver() { + RTCRtpTransceiver[] transceivers = callee.getPeerConnection().getTransceivers(); + RTCRtpReceiver receiver = transceivers[0].getReceiver(); + + for (RTCRtpTransceiver transceiver : transceivers) { + transceiver.dispose(); + } + + return receiver; + } + + Map awaitMediaSourceStats(Predicate> condition) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (System.nanoTime() < deadline) { + CountDownLatch done = new CountDownLatch(1); + AtomicReference> found = new AtomicReference<>(); + + caller.getPeerConnection().getStats(sender, report -> { + for (RTCStats stats : report.getStats().values()) { + if (stats.getType() == RTCStatsType.MEDIA_SOURCE && condition.test(stats.getAttributes())) { + found.set(stats.getAttributes()); + } + } + done.countDown(); + }); + + done.await(5, TimeUnit.SECONDS); + + if (found.get() != null) { + return found.get(); + } + + Thread.sleep(50); + } + + return null; + } + + @Override + public void close() { + sender.dispose(); + + caller.close(); + callee.close(); + + track.dispose(); + source.dispose(); + factory.dispose(); + audioModule.dispose(); + } + + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/ToneAudioDeviceModule.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/ToneAudioDeviceModule.java new file mode 100644 index 000000000..2a4aa1b51 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/ToneAudioDeviceModule.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +/** + * An audio device module without audio hardware that records a sine tone, 48 + * kHz mono, while WebRTC records from it. It stands in for a microphone, so + * that the tests can send device-captured audio. It lives in the tests so that + * nothing else can reach it. + * + * @author Alex Andres + */ +public class ToneAudioDeviceModule extends AudioDeviceModuleBase { + + /** + * Creates a module that records a tone of the given frequency. + * + * @param frequencyHz The frequency of the recorded tone. + */ + public ToneAudioDeviceModule(int frequencyHz) { + initialize(frequencyHz); + } + + private native void initialize(int frequencyHz); + +}