From b306087dfb7f5b4351ecfe017a0276f94fa8a440 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 8 Nov 2026 13:03:27 +0100 Subject: [PATCH 1/3] feat: add system audio capture sources for desktop sharing --- docs/.vitepress/sidebar.ts | 1 + docs/guide/audio/system-audio-source.md | 112 +++++++++ docs/guide/video/desktop-capture.md | 8 +- .../webrtc/examples/SystemAudioExample.java | 56 +++++ webrtc-jni/src/main/cpp/CMakeLists.txt | 2 +- .../main/cpp/include/JNI_SystemAudioSource.h | 25 +++ .../include/media/audio/SystemAudioCapture.h | 50 +++++ .../media/audio/linux/PulseAudioLoader.h | 31 +++ .../main/cpp/src/JNI_SystemAudioSource.cpp | 47 ++++ .../src/media/audio/SystemAudioCapture.cpp | 77 +++++++ .../media/audio/linux/SystemAudioCapture.cpp | 151 +++++++++++++ .../media/audio/macos/SystemAudioCapture.cpp | 191 ++++++++++++++++ .../audio/windows/SystemAudioCapture.cpp | 99 ++++++++ .../webrtc/media/audio/SystemAudioSource.java | 212 ++++++++++++++++++ .../SystemAudioSourceIntegrationTest.java | 58 +++++ .../media/audio/SystemAudioSourceTest.java | 195 ++++++++++++++++ 16 files changed, 1313 insertions(+), 2 deletions(-) create mode 100644 docs/guide/audio/system-audio-source.md create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java create mode 100644 webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h create mode 100644 webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h create mode 100644 webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 9d7b3d913..6613c3863 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -35,6 +35,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { { text: 'Audio Devices', link: '/audio/audio-devices' }, { text: 'Audio Processing', link: '/audio/audio-processing' }, { text: 'Custom Audio Source', link: '/audio/custom-audio-source' }, + { text: 'System Audio Source', link: '/audio/system-audio-source' }, { text: 'Headless Audio', link: '/audio/headless-audio' }, { text: 'DTMF Sender', link: '/audio/dtmf-sender' }, ], diff --git a/docs/guide/audio/system-audio-source.md b/docs/guide/audio/system-audio-source.md new file mode 100644 index 000000000..c4184f3a6 --- /dev/null +++ b/docs/guide/audio/system-audio-source.md @@ -0,0 +1,112 @@ +# System Audio Source + +`SystemAudioSource` captures audio playing on the system output. It is a +`CustomAudioSource` and can be passed directly to `PeerConnectionFactory.createAudioTrack`. +It sends 16-bit, 48 kHz stereo PCM in 10 ms chunks, independently of screen or +window video capture. Capturing a window does **not** restrict audio to that window. + +## Platform support + +| Platform | Capture backend | Requirements and scope | +| --- | --- | --- | +| Windows | WASAPI shared-mode loopback | Captures the default multimedia output selected at start, including audio from this process. | +| Linux | PulseAudio output monitor | Requires `libpulse.so.0` and PulseAudio or PipeWire with `pipewire-pulse`. Captures the default sink selected at start, including audio from this process. A native PipeWire portal audio backend is not provided. | +| macOS 13+ | ScreenCaptureKit | Requires screen/audio recording permission and a shareable display. Captures system audio excluding this process. Build with a macOS 13+ SDK. | + +No shell commands, virtual audio driver, or FFmpeg installation are required. +There is no application/process selection API in this version. Protected content +and platform recording restrictions still apply. + +## Send alongside a screen track + +Use your existing factory and peer connection, before negotiation: + +```java +import dev.onvoid.webrtc.RTCRtpSender; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.SystemAudioSource; +import java.util.Collections; + +SystemAudioSource systemAudio = new SystemAudioSource(); +AudioTrack audioTrack = factory.createAudioTrack("presentation-audio", systemAudio); +RTCRtpSender audioSender = peerConnection.addTrack( + audioTrack, Collections.singletonList("presentation")); +// Add your VideoDesktopSource's video track with the same stream ID. +systemAudio.start(); +``` + +`start()` waits for capture to open and throws `IllegalStateException` on startup +failure, with the platform error as its cause. Call it on a background thread, +particularly when macOS may show a permission prompt. Platform startup waits are +bounded; permission dialogs can remain open after a timeout. A late macOS grant +will not leave this timed-out capture recording in the background. Grant permission +in System Settings and retry, restarting the application if the OS requires it. + +When the call ends, stop recording before releasing tracks and the source: + +```java +systemAudio.stop(); +peerConnection.removeTrack(audioSender); +audioSender.dispose(); +audioTrack.dispose(); +systemAudio.dispose(); // or close(), also supported by try-with-resources +``` + +Use `try/finally` in application code to release these resources on errors too. +`stop()` and `dispose()` are idempotent. `stop()` keeps the source and its tracks +usable for `start()` again; disposal is permanent. A stop or disposal from a local +audio sink requests shutdown without joining its own callback thread. Do not +restart from an audio callback or push additional PCM into this source. + +## Device and permission changes + +Observe `isRecording()` and `getFailure()` from your application control thread. +A runtime capture failure stops recording and retains its cause in `getFailure()`; +starting again clears that failure. A stopped source does not end its tracks, so +it can resume on the same sender. It does not automatically reopen a device or +silently change recording permissions. + +For an output change, call `stop()`, select the new default output through the OS, +and call `start()`. A change of the default output alone does not move a running +capture: Windows and Linux bind the output chosen at startup. Device loss or +permission revocation may only become observable when the platform reports it. + +## Synchronization and latency + +Capture timestamps use WebRTC's monotonic clock, the same clock as +`SyncClock.currentTimeUs()` and desktop video. WASAPI's packet clock and +ScreenCaptureKit's presentation clock are translated into that clock; PulseAudio +uses its reported capture latency. This preserves the capture time when audio is +buffered before delivery. Queueing is bounded to 100 ms; older chunks are dropped +if delivery stalls. Audio is paced by the OS rather than an independent Java timer. +This does not promise sample-accurate lip synchronization or compensate for all +screen capture, output-device, and receiver latency. + +## Microphone and feedback + +A factory that sends pushed audio cannot also send its own device-captured audio +from `factory.createAudioSource(...)`. The existing factory guard applies to +`SystemAudioSource` too. For a microphone alongside system audio on the same +connection, capture the microphone independently (for example with +`AudioRecorder`) and feed a **separate** `CustomAudioSource`, so both tracks use +pushed audio. Respect that source's 16-bit PCM and 10 ms chunk requirements. Stop +the recorder before disposing its custom source. Alternatively, use separate +factories and connections for device-captured microphone and system audio. + +No microphone mixing or echo cancellation is added here. Keep microphone and +system audio on separate tracks, and use headphones to prevent acoustic feedback. +On Windows and Linux, route received call audio to another output or to a virtual +output that is not monitored. Headphones alone do not prevent a loopback source +from capturing remote audio played on the monitored output. On macOS this process's +playout is excluded by ScreenCaptureKit; remote audio played in another process +can still be captured. + +Run `SystemAudioExample` from `webrtc-examples` for a capture-only example that +counts received PCM frames without playing them back. The opt-in +`SystemAudioSourceIntegrationTest` (`-Dtest.systemAudio=true`) needs audio playing +on the default output and recording permission; lifecycle tests use deterministic +backends without hardware. + +The platform bindings follow [Microsoft's loopback capture API](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording), +[PulseAudio's recording stream API](https://www.freedesktop.org/software/pulseaudio/doxygen/stream_8h.html), +and [ScreenCaptureKit audio configuration](https://developer.apple.com/documentation/screencapturekit/scstreamconfiguration/capturesaudio). diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index 4207121b9..da9127079 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -189,4 +189,10 @@ When implementing desktop capture in your application, remember to: - Consider the performance implications of higher resolutions and frame rates - Handle potential permission requirements on different operating systems -Desktop capture is particularly useful for screen sharing applications, remote assistance tools, collaborative workspaces, and educational platforms where visual content sharing is essential. \ No newline at end of file +Desktop capture is particularly useful for screen sharing applications, remote assistance tools, collaborative workspaces, and educational platforms where visual content sharing is essential. + +## Sharing system audio + +Add a separate [`SystemAudioSource`](/guide/audio/system-audio-source) audio track +to share output audio alongside desktop video. Window selection applies only to +video; the audio source captures system output independently. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java new file mode 100644 index 000000000..c7c5ef9b2 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.SystemAudioSource; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Captures the default system output into a native audio track without playing + * it back. Play audio in another application and press Enter to stop. + * For sending, add this track beside the desktop video track on a peer connection. + */ +public class SystemAudioExample { + public static void main(String[] args) throws Exception { + // SystemAudioSource opens its own output capture; no microphone is needed. + try (AudioDeviceModule audio = new AudioDeviceModule(AudioLayer.kDummyAudio); + PeerConnectionFactory factory = new PeerConnectionFactory(audio); + SystemAudioSource source = new SystemAudioSource()) { + AudioTrack track = factory.createAudioTrack("system-audio", source); + AtomicLong frames = new AtomicLong(); + try { + track.addSink((data, bits, rate, channels, count) -> frames.addAndGet(count)); + source.start(); + System.out.println("Capturing system audio. Play audio in another app; press Enter to stop."); + System.in.read(); + source.stop(); + if (source.getFailure() != null) { + throw new IllegalStateException("System audio capture failed", source.getFailure()); + } + System.out.println("Captured " + frames.get() + " stereo PCM frames"); + } + finally { + source.stop(); + track.dispose(); + } + } + } +} diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index 1befe80b6..29a50daa6 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -139,7 +139,7 @@ target_link_libraries(${PROJECT_NAME} webrtc) if(APPLE) set_source_files_properties(${SOURCES} PROPERTIES COMPILE_FLAGS "-x objective-c++") target_link_options(${PROJECT_NAME} PRIVATE "-ObjC") - target_link_libraries(${PROJECT_NAME} "-framework Foundation" "-framework AVFoundation" "-framework CoreMedia" "-framework CoreAudio" "-framework IOKit" "-framework CoreVideo" "-framework VideoToolbox" "-framework QuartzCore") + target_link_libraries(${PROJECT_NAME} "-weak_framework ScreenCaptureKit" "-framework Foundation" "-framework AVFoundation" "-framework CoreMedia" "-framework CoreAudio" "-framework IOKit" "-framework CoreVideo" "-framework VideoToolbox" "-framework QuartzCore") elseif(LINUX) set(CXX_LIBS "-static-libgcc -stdlib=libc++ -lc++ -lc++abi") diff --git a/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h b/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h new file mode 100644 index 000000000..7f143907f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_SYSTEM_AUDIO_SOURCE_H +#define JNI_SYSTEM_AUDIO_SOURCE_H +#include +extern "C" { +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_initialize(JNIEnv *, jobject); +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_pump(JNIEnv *, jobject, jobject); +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_dispose(JNIEnv *, jobject); +} +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h b/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h new file mode 100644 index 000000000..87686fbd9 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_SYSTEM_AUDIO_CAPTURE_H +#define JNI_SYSTEM_AUDIO_CAPTURE_H + +#include "media/audio/CustomAudioSource.h" +#include +#include +#include +#include +#include + +namespace jni { +// Platform callbacks only write PCM into a bounded queue; they never call Java +// or WebRTC. The owning Java worker polls and delivers complete 10 ms chunks. +class SystemAudioCapture { +public: + virtual ~SystemAudioCapture() = default; + virtual void Open() = 0; + virtual void Poll() = 0; + void Pump(CustomAudioSource * source); + void Append(const int16_t * samples, size_t frames, int64_t timestamp_us); + void Fail(const char * message) noexcept; + static std::unique_ptr Create(); +private: + struct Chunk { + std::array samples{}; + size_t frames = 0; + int64_t time = 0; + }; + std::mutex mutex_; + std::deque chunks_; + std::string failure_; +}; +} +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h index 1ec01f491..cdfdd0a57 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h @@ -22,6 +22,7 @@ #include #include +#include #include "Exception.h" @@ -68,6 +69,7 @@ namespace jni } bool load() { + std::lock_guard lock(load_mutex); if (loaded) return true; // Try to open the library @@ -110,6 +112,24 @@ namespace jni LOAD_SYM(pa_operation_get_state); LOAD_SYM(pa_proplist_gets); + pa_stream_new = reinterpret_cast(dlsym(lib_handle, "pa_stream_new")); + if (!pa_stream_new) { close(); return false; } + pa_stream_unref = reinterpret_cast(dlsym(lib_handle, "pa_stream_unref")); + if (!pa_stream_unref) { close(); return false; } + pa_stream_connect_record = reinterpret_cast(dlsym(lib_handle, "pa_stream_connect_record")); + if (!pa_stream_connect_record) { close(); return false; } + pa_stream_disconnect = reinterpret_cast(dlsym(lib_handle, "pa_stream_disconnect")); + if (!pa_stream_disconnect) { close(); return false; } + pa_stream_get_state = reinterpret_cast(dlsym(lib_handle, "pa_stream_get_state")); + if (!pa_stream_get_state) { close(); return false; } + pa_stream_peek = reinterpret_cast(dlsym(lib_handle, "pa_stream_peek")); + if (!pa_stream_peek) { close(); return false; } + pa_stream_drop = reinterpret_cast(dlsym(lib_handle, "pa_stream_drop")); + if (!pa_stream_drop) { close(); return false; } + pa_stream_readable_size = reinterpret_cast(dlsym(lib_handle, "pa_stream_readable_size")); + if (!pa_stream_readable_size) { close(); return false; } + pa_stream_get_latency = reinterpret_cast(dlsym(lib_handle, "pa_stream_get_latency")); + if (!pa_stream_get_latency) { close(); return false; } loaded = true; return true; } @@ -156,7 +176,18 @@ namespace jni pa_operation_get_state_t pa_operation_get_state; pa_proplist_gets_t pa_proplist_gets; + decltype(&::pa_stream_new) pa_stream_new = nullptr; + decltype(&::pa_stream_unref) pa_stream_unref = nullptr; + decltype(&::pa_stream_connect_record) pa_stream_connect_record = nullptr; + decltype(&::pa_stream_disconnect) pa_stream_disconnect = nullptr; + decltype(&::pa_stream_get_state) pa_stream_get_state = nullptr; + decltype(&::pa_stream_peek) pa_stream_peek = nullptr; + decltype(&::pa_stream_drop) pa_stream_drop = nullptr; + decltype(&::pa_stream_readable_size) pa_stream_readable_size = nullptr; + decltype(&::pa_stream_get_latency) pa_stream_get_latency = nullptr; + private: + std::mutex load_mutex; PulseAudioLoader() : loaded(false), lib_handle(nullptr) {} bool loaded; void* lib_handle; diff --git a/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp new file mode 100644 index 000000000..b4b32d028 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_SystemAudioSource.h" +#include "JavaBoundary.h" +#include "NativeHandle.h" +#include "media/audio/SystemAudioCapture.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_initialize +(JNIEnv * env, jobject caller) { + jni::CallFromJava(env, [&]() { + auto capture = jni::SystemAudioCapture::Create(); + capture->Open(); + SetHandle(env, caller, capture.get()); + capture.release(); + }); +} +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_pump +(JNIEnv * env, jobject caller, jobject javaSource) { + jni::CallFromJava(env, [&]() { + jni::NativeLease capture(env, caller); + CHECK_HANDLE(capture); + jni::NativeLease source(env, javaSource); + CHECK_HANDLE(source); + capture->Pump(source); + }); +} +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_dispose +(JNIEnv * env, jobject caller) { + jni::CallFromJava(env, [&]() { + auto * capture = jni::TakeHandle(env, caller); + if (capture) jni::ReleaseHandle(env, capture, [capture](bool) { delete capture; }); + }); +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp new file mode 100644 index 000000000..7bc58d670 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/SystemAudioCapture.h" +#include "Exception.h" +#include +#include + +namespace jni { +void SystemAudioCapture::Append(const int16_t * samples, size_t frames, int64_t time) { + std::lock_guard lock(mutex_); + // Do not splice a partial chunk from before an output pause into new + // audio. The gap must remain a timestamp gap on the outgoing stream. + if (!chunks_.empty() && chunks_.back().frames < 480) { + const auto & tail = chunks_.back(); + const int64_t expected = tail.time + tail.frames * 1000000 / 48000; + if (time - expected > 5000 || expected - time > 5000) chunks_.pop_back(); + } + size_t offset = 0; + while (offset < frames) { + if (chunks_.empty() || chunks_.back().frames == 480) { + // Bound latency and memory if the consumer stalls. Preserve capture + // timestamps when dropping old audio instead of replaying a backlog. + if (chunks_.size() >= 10) chunks_.pop_front(); + chunks_.emplace_back(); + chunks_.back().time = time + offset * 1000000 / 48000; + } + auto & chunk = chunks_.back(); + const size_t count = std::min(frames - offset, 480 - chunk.frames); + if (samples) { + std::memcpy(chunk.samples.data() + chunk.frames * 2, + samples + offset * 2, count * 4); + } + else { + std::memset(chunk.samples.data() + chunk.frames * 2, 0, count * 4); + } + chunk.frames += count; + offset += count; + } +} +void SystemAudioCapture::Fail(const char * message) noexcept { + // Native OS callbacks must never unwind through platform code. + try { + std::lock_guard lock(mutex_); + failure_ = message; + } + catch (...) { } +} +void SystemAudioCapture::Pump(CustomAudioSource * source) { + Poll(); + std::deque ready; + { + std::lock_guard lock(mutex_); + if (!failure_.empty()) throw Exception(failure_.c_str()); + while (!chunks_.empty() && chunks_.front().frames == 480) { + ready.push_back(std::move(chunks_.front())); + chunks_.pop_front(); + } + } + for (auto & chunk : ready) { + source->PushAudioData(chunk.samples.data(), 16, 48000, 2, 480, chunk.time); + } +} +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp new file mode 100644 index 000000000..0b8e695c4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp @@ -0,0 +1,151 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/SystemAudioCapture.h" +#include "media/audio/linux/PulseAudioLoader.h" +#include "Exception.h" +#include "rtc_base/time_utils.h" +#include +#include + +namespace jni { +namespace { +class PulseCapture final : public SystemAudioCapture { + avdev::PulseAudioLoader & pa = avdev::PulseAudioLoader::instance(); + pa_threaded_mainloop * loop = nullptr; + pa_context * context = nullptr; + pa_stream * stream = nullptr; + bool started = false; + std::string sink, monitor; + bool infoDone = false; + struct Lock { + PulseCapture & self; + explicit Lock(PulseCapture & s) : self(s) { self.pa.pa_threaded_mainloop_lock(self.loop); } + ~Lock() { self.pa.pa_threaded_mainloop_unlock(self.loop); } + }; + template void Await(Test test) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + for (;;) { + { + Lock lock(*this); + if (!PA_CONTEXT_IS_GOOD(pa.pa_context_get_state(context))) + throw Exception("System audio: PulseAudio connection lost"); + if (test()) return; + } + if (std::chrono::steady_clock::now() >= deadline) + throw Exception("System audio: PulseAudio startup timed out"); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + } +public: + ~PulseCapture() override { + if (loop && started) { + { + Lock lock(*this); + if (stream) { pa.pa_stream_disconnect(stream); pa.pa_stream_unref(stream); } + if (context) { pa.pa_context_disconnect(context); pa.pa_context_unref(context); } + } + pa.pa_threaded_mainloop_stop(loop); + } + if (loop) pa.pa_threaded_mainloop_free(loop); + } + void Open() override { + if (!pa.load()) throw Exception("System audio requires libpulse.so.0"); + loop = pa.pa_threaded_mainloop_new(); + if (!loop) throw Exception("System audio: cannot create PulseAudio mainloop"); + if (pa.pa_threaded_mainloop_start(loop) < 0) + throw Exception("System audio: cannot start PulseAudio mainloop"); + started = true; + { + Lock lock(*this); + context = pa.pa_context_new(pa.pa_threaded_mainloop_get_api(loop), "webrtc-java system audio"); + if (!context || pa.pa_context_connect(context, nullptr, PA_CONTEXT_NOAUTOSPAWN, nullptr) < 0) + throw Exception("System audio: cannot connect to PulseAudio / pipewire-pulse"); + } + Await([&] { return pa.pa_context_get_state(context) == PA_CONTEXT_READY; }); + { + Lock lock(*this); + auto * op = pa.pa_context_get_server_info(context, + [](pa_context *, const pa_server_info * info, void * data) noexcept { + auto * self = static_cast(data); + try { if (info && info->default_sink_name) self->sink = info->default_sink_name; } + catch (...) { self->Fail("System audio: cannot read default sink"); } + self->infoDone = true; + }, this); + if (!op) throw Exception("System audio: cannot query default output"); + pa.pa_operation_unref(op); + } + Await([&] { return infoDone; }); + { + Lock lock(*this); + if (sink.empty()) throw Exception("System audio: no default output"); + infoDone = false; + auto * op = pa.pa_context_get_sink_info_by_name(context, sink.c_str(), + [](pa_context *, const pa_sink_info * info, int end, void * data) noexcept { + auto * self = static_cast(data); + try { if (info && info->monitor_source_name) self->monitor = info->monitor_source_name; } + catch (...) { self->Fail("System audio: cannot read output monitor"); } + if (end) self->infoDone = true; + }, this); + if (!op) throw Exception("System audio: cannot query output monitor"); + pa.pa_operation_unref(op); + } + Await([&] { return infoDone; }); + { + Lock lock(*this); + if (monitor.empty()) throw Exception("System audio: output has no monitor source"); + pa_sample_spec spec{PA_SAMPLE_S16NE, 48000, 2}; + stream = pa.pa_stream_new(context, "System output", &spec, nullptr); + if (!stream) throw Exception("System audio: cannot create monitor stream"); + pa_buffer_attr attr{19200, UINT32_MAX, UINT32_MAX, UINT32_MAX, 1920}; + if (pa.pa_stream_connect_record(stream, monitor.c_str(), &attr, + static_cast(PA_STREAM_ADJUST_LATENCY | PA_STREAM_DONT_MOVE)) < 0) + throw Exception("System audio: cannot open output monitor"); + } + Await([&] { + const auto state = pa.pa_stream_get_state(stream); + if (!PA_STREAM_IS_GOOD(state)) throw Exception("System audio: monitor unavailable or permission denied"); + return state == PA_STREAM_READY; + }); + } + void Poll() override { + Lock lock(*this); + if (pa.pa_context_get_state(context) != PA_CONTEXT_READY || pa.pa_stream_get_state(stream) != PA_STREAM_READY) + throw Exception("System audio: output lost; stop and restart after selecting an output"); + // A bounded drain keeps stop responsive even if an output is very busy. + for (int i = 0; i < 10; ++i) { + size_t available = pa.pa_stream_readable_size(stream); + if (available == static_cast(-1)) throw Exception("System audio: monitor read failed"); + if (!available) break; + const void * data = nullptr; + size_t bytes = 0; + if (pa.pa_stream_peek(stream, &data, &bytes) < 0) throw Exception("System audio: monitor peek failed"); + if (!bytes) break; + pa_usec_t latency = 0; + int negative = 0; + if (pa.pa_stream_get_latency(stream, &latency, &negative) < 0 || negative) latency = 0; + try { + Append(static_cast(data), bytes / 4, + webrtc::TimeMicros() - static_cast(latency)); + } + catch (...) { pa.pa_stream_drop(stream); throw; } + if (pa.pa_stream_drop(stream) < 0) throw Exception("System audio: monitor drop failed"); + } + } +}; +} +std::unique_ptr SystemAudioCapture::Create() { return std::make_unique(); } +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp new file mode 100644 index 000000000..06715dd4b --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp @@ -0,0 +1,191 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/SystemAudioCapture.h" +#include "Exception.h" +#include "rtc_base/time_utils.h" +#import +#import +#include +#include +#include +#include +#include +#include + +// A receiver may outlive a timed-out start/stop. Detach its owner under the +// same lock used by callbacks before destroying the native capture object. +API_AVAILABLE(macos(13.0)) +@interface SystemAudioReceiver : NSObject { +@public + jni::SystemAudioCapture * owner; +} +@end + +@implementation SystemAudioReceiver +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + @synchronized(self) { + if (owner) owner->Fail("System audio: ScreenCaptureKit stopped; check recording permission and restart"); + } +} +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sample ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeAudio || !CMSampleBufferDataIsReady(sample)) return; + @synchronized(self) { + if (!owner) return; + try { + const auto * format = CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sample)); + if (!format || format->mFormatID != kAudioFormatLinearPCM || + format->mSampleRate != 48000 || format->mChannelsPerFrame != 2 || + !(format->mFormatFlags & kAudioFormatFlagIsFloat) || format->mBitsPerChannel != 32) { + owner->Fail("System audio: unexpected ScreenCaptureKit PCM format"); + return; + } + struct { AudioBufferList list; AudioBuffer extra; } storage{}; + CMBlockBufferRef block = nullptr; + OSStatus status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sample, nullptr, + &storage.list, sizeof(storage), kCFAllocatorDefault, kCFAllocatorDefault, + kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &block); + // Release the retained block even if allocating the conversion buffer fails. + std::unique_ptr retained(block, &CFRelease); + if (status != noErr) { owner->Fail("System audio: cannot read ScreenCaptureKit PCM"); return; } + const size_t frames = CMSampleBufferGetNumSamples(sample); + const bool planar = format->mFormatFlags & kAudioFormatFlagIsNonInterleaved; + const auto & list = storage.list; + if (list.mNumberBuffers != (planar ? 2U : 1U)) { + owner->Fail("System audio: unexpected ScreenCaptureKit channel layout"); return; + } + for (UInt32 b = 0; b < list.mNumberBuffers; ++b) { + if (!list.mBuffers[b].mData || list.mBuffers[b].mDataByteSize < frames * sizeof(float) * (planar ? 1 : 2)) { + owner->Fail("System audio: incomplete ScreenCaptureKit PCM"); return; + } + } + std::vector pcm(frames * 2); + for (size_t f = 0; f < frames; ++f) { + for (size_t c = 0; c < 2; ++c) { + float value = static_cast(list.mBuffers[planar ? c : 0].mData)[planar ? f : f * 2 + c]; + if (!std::isfinite(value)) value = 0; + pcm[f * 2 + c] = static_cast(std::max(-32768.0f, std::min(32767.0f, value * 32768.0f))); + } + } + const double pts = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sample)); + const double now = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())); + if (!std::isfinite(pts) || !std::isfinite(now)) { + owner->Fail("System audio: invalid ScreenCaptureKit timestamp"); return; + } + owner->Append(pcm.data(), frames, webrtc::TimeMicros() + static_cast((pts - now) * 1000000)); + } + catch (...) { owner->Fail("System audio: ScreenCaptureKit callback failed"); } + } +} +@end + +namespace jni { +namespace { +struct Completion { + std::mutex mutex; + std::condition_variable condition; + bool done = false; + std::atomic abandoned{false}; + id keepAlive = nil; + NSError * error = nil; + SCShareableContent * content = nil; + ~Completion() { [keepAlive release]; [error release]; [content release]; } + void Finish(NSError * e, SCShareableContent * c = nil) noexcept { + try { + std::lock_guard lock(mutex); + error = [e retain]; content = [c retain]; done = true; + condition.notify_all(); + } + catch (...) { } // The waiting caller will time out; never unwind into the OS. + } + void Wait() { + std::unique_lock lock(mutex); + if (!condition.wait_for(lock, std::chrono::seconds(15), [&] { return done; })) { + abandoned = true; + throw Exception("System audio: ScreenCaptureKit timed out; check screen/audio recording permission"); + } + if (error) throw Exception([[error localizedDescription] UTF8String]); + } +}; +class API_AVAILABLE(macos(13.0)) ScreenAudioCapture final : public SystemAudioCapture { + SCStream * stream = nil; + SystemAudioReceiver * receiver = nil; + dispatch_queue_t queue = nullptr; +public: + ~ScreenAudioCapture() override { + @autoreleasepool { + if (receiver) { @synchronized(receiver) { receiver->owner = nullptr; } } + if (stream) { + auto completed = std::make_shared(); + completed->keepAlive = [receiver retain]; + [stream stopCaptureWithCompletionHandler:^(NSError * error) { completed->Finish(error); }]; + try { completed->Wait(); } catch (...) { } + [stream release]; + } + [receiver release]; + if (queue) dispatch_release(queue); + } + } + void Open() override { + @autoreleasepool { + auto content = std::make_shared(); + [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:YES + completionHandler:^(SCShareableContent * result, NSError * error) { content->Finish(error, result); }]; + content->Wait(); + if (!content->content || content->content.displays.count == 0) + throw Exception("System audio: no shareable display; grant screen/audio recording permission"); + SCContentFilter * filter = [[SCContentFilter alloc] initWithDisplay:content->content.displays.firstObject excludingWindows:@[]]; + SCStreamConfiguration * config = [[SCStreamConfiguration alloc] init]; + config.capturesAudio = YES; + config.excludesCurrentProcessAudio = YES; + config.sampleRate = 48000; + config.channelCount = 2; + config.width = 2; config.height = 2; + config.minimumFrameInterval = CMTimeMake(1, 1); + receiver = [[SystemAudioReceiver alloc] init]; + receiver->owner = this; + queue = dispatch_queue_create("webrtc-system-audio", DISPATCH_QUEUE_SERIAL); + stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:receiver]; + [filter release]; [config release]; + NSError * error = nil; + if (![stream addStreamOutput:receiver type:SCStreamOutputTypeAudio sampleHandlerQueue:queue error:&error]) + throw Exception(error ? [[error localizedDescription] UTF8String] : "System audio: cannot add ScreenCaptureKit audio output"); + auto completed = std::make_shared(); + completed->keepAlive = [receiver retain]; + SCStream * starting = stream; + [stream startCaptureWithCompletionHandler:^(NSError * error) { + // A permission prompt can outlive the caller's deadline. A late + // successful start must not leave a recording running invisibly. + if (completed->abandoned.load() && !error) { + // Keep the detached receiver alive until the late stream + // has stopped, even when the original stop timed out too. + [starting stopCaptureWithCompletionHandler:^(NSError * stopError) { + completed->Finish(stopError); + }]; + } + else completed->Finish(error); + }]; + completed->Wait(); + } + } + void Poll() override { } +}; +} +std::unique_ptr SystemAudioCapture::Create() { + if (@available(macOS 13.0, *)) return std::make_unique(); + throw Exception("System audio requires macOS 13 or later"); +} +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp new file mode 100644 index 000000000..8ecb087ae --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/audio/SystemAudioCapture.h" +#include "Exception.h" +#include "rtc_base/time_utils.h" +#include +#include +#include +#include +#include + +namespace jni { +namespace { +using Microsoft::WRL::ComPtr; +void Check(HRESULT hr, const char * operation) { + if (FAILED(hr)) { + std::ostringstream message; + message << "System audio: " << operation << " (HRESULT 0x" << std::hex + << static_cast(hr) << "); restart capture after an output change"; + throw Exception(message.str().c_str()); + } +} +class WasapiCapture final : public SystemAudioCapture { + bool com = false; + ComPtr enumerator; + ComPtr device; + ComPtr client; + ComPtr capture; +public: + ~WasapiCapture() override { + if (client) client->Stop(); + capture.Reset(); client.Reset(); device.Reset(); enumerator.Reset(); + if (com) CoUninitialize(); + } + void Open() override { + Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "initialize COM"); + com = true; + Check(CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, + IID_PPV_ARGS(enumerator.GetAddressOf())), "create output enumerator"); + Check(enumerator->GetDefaultAudioEndpoint(eRender, eMultimedia, device.GetAddressOf()), "get default output"); + Check(device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, + reinterpret_cast(client.GetAddressOf())), "activate output"); + WAVEFORMATEX format{}; + format.wFormatTag = WAVE_FORMAT_PCM; + format.nChannels = 2; + format.nSamplesPerSec = 48000; + format.wBitsPerSample = 16; + format.nBlockAlign = 4; + format.nAvgBytesPerSec = 192000; + Check(client->Initialize(AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | + AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, + 1000000, 0, &format, nullptr), "initialize loopback"); + Check(client->GetService(IID_PPV_ARGS(capture.GetAddressOf())), "get loopback capture"); + Check(client->Start(), "start loopback"); + } + void Poll() override { + for (int i = 0; i < 10; ++i) { + UINT32 frames = 0; + Check(capture->GetNextPacketSize(&frames), "query output packet"); + if (!frames) break; + BYTE * data = nullptr; + DWORD flags = 0; + UINT64 position = 0, qpc = 0; + Check(capture->GetBuffer(&data, &frames, &flags, &position, &qpc), "read output packet"); + try { + int64_t time = webrtc::TimeMicros() - frames * 1000000LL / 48000; + if (!(flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR)) { + LARGE_INTEGER now, frequency; + QueryPerformanceCounter(&now); + QueryPerformanceFrequency(&frequency); + const int64_t host_us = static_cast( + static_cast(now.QuadPart) * 1000000 / frequency.QuadPart); + time = webrtc::TimeMicros() + static_cast(qpc / 10) - host_us; + } + Append(flags & AUDCLNT_BUFFERFLAGS_SILENT ? nullptr : reinterpret_cast(data), frames, time); + } + catch (...) { capture->ReleaseBuffer(frames); throw; } + Check(capture->ReleaseBuffer(frames), "release output packet"); + } + } +}; +} +std::unique_ptr SystemAudioCapture::Create() { return std::make_unique(); } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java new file mode 100644 index 000000000..1817d739b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java @@ -0,0 +1,212 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import dev.onvoid.webrtc.internal.DisposableNativeObject; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import java.util.concurrent.locks.LockSupport; + +/** + * Captures the system output as a separate audio track source, at 48 kHz stereo. + * Use with {@code PeerConnectionFactory.createAudioTrack}, alongside a screen + * video track. A separate microphone track must also use pushed audio; + * do not combine this with the factory's device-captured audio sources. + *

+ * Windows uses WASAPI loopback of the default multimedia output; Linux uses + * the default output's PulseAudio monitor (also available with pipewire-pulse). + * macOS 13 or later uses ScreenCaptureKit and excludes this process's audio. + * Screen/audio recording permission is required on macOS. This class does not + * capture a selected window's or process's audio. + *

+ * Call {@link #start()} to begin and {@link #stop()} before changing output + * devices. A restart selects the current default output. Capture errors stop + * recording and are available through {@link #getFailure()}. Remote call audio + * played on the captured output is included on Windows and Linux: route it to + * another output to prevent feedback. No microphone mixing or echo cancellation + * is performed. Do not call the inherited pushAudio methods during capture. + */ +public final class SystemAudioSource extends CustomAudioSource implements AutoCloseable { + + private volatile boolean recording; + private volatile Throwable failure; + private volatile Thread worker; + private volatile boolean disposed; + private final AtomicBoolean released = new AtomicBoolean(); + private final Supplier backend; + + /** Creates an idle source. No device or recording permission is opened yet. */ + public SystemAudioSource() { + this(Capture::new); + } + + SystemAudioSource(Supplier backend) { + this.backend = backend; + } + + /** + * Starts capture, waiting for the platform to open the output. + * Calling this while already recording does nothing. Startup uses bounded platform waits, + * including the macOS permission prompt; use a background thread. + * + * @throws IllegalStateException if disposed or capture cannot start + */ + public void start() { + if (Thread.currentThread() == worker) { + throw new IllegalStateException("Start capture outside an audio callback"); + } + synchronized (this) { + startLocked(); + } + } + + private void startLocked() { + if (disposed) { + throw new IllegalStateException("System audio source is disposed"); + } + if (recording) { + return; + } + stopLocked(); + // The previous worker may have disposed the source from its last sink + // callback while stopLocked was joining it. + if (disposed) { + throw new IllegalStateException("System audio source is disposed"); + } + failure = null; + CompletableFuture started = new CompletableFuture<>(); + recording = true; + worker = new Thread(() -> capture(started), "webrtc-system-audio"); + worker.setDaemon(true); + try { + worker.start(); + started.join(); + } + catch (CompletionException e) { + stop(); + throw new IllegalStateException("Cannot start system audio capture", e.getCause()); + } + catch (RuntimeException | Error e) { + recording = false; + throw e; + } + } + + private void capture(CompletableFuture started) { + try (Backend capture = backend.get()) { + capture.initialize(); + started.complete(null); + while (recording) { + capture.pump(this); + LockSupport.parkNanos(2_000_000L); + } + } + catch (Throwable e) { + failure = e; + started.completeExceptionally(e); + } + finally { + recording = false; + if (disposed) releaseSource(); + } + } + + /** Stops and joins capture; the source and its tracks may be reused. */ + public void stop() { + // A local track sink can run inside pump(). It must not wait for its + // own worker or for a lifecycle caller that is joining that worker. + if (Thread.currentThread() == worker) { + recording = false; + return; + } + synchronized (this) { + stopLocked(); + } + } + + private void stopLocked() { + recording = false; + boolean interrupted = false; + if (worker != null) { + LockSupport.unpark(worker); + while (worker.isAlive()) { + try { + worker.join(); + } + catch (InterruptedException e) { + interrupted = true; + } + } + worker = null; + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** @return whether capture is running (including startup) */ + public boolean isRecording() { + return recording; + } + + /** @return the last startup/runtime failure, or null; cleared by start */ + public Throwable getFailure() { + return failure; + } + + /** Stops capture before releasing the native source. Safe to call repeatedly. */ + @Override + public void dispose() { + if (Thread.currentThread() == worker) { + disposed = true; + recording = false; + return; + } + synchronized (this) { + disposed = true; + stopLocked(); + releaseSource(); + } + } + + /** Stops capture and releases the source. */ + @Override + public void close() { + dispose(); + } + + private void releaseSource() { + if (released.compareAndSet(false, true)) super.dispose(); + } + + interface Backend extends AutoCloseable { + void initialize(); + void pump(CustomAudioSource source); + void dispose(); + default void close() { dispose(); } + } + + // Open, poll and close on the same thread (required by WASAPI's COM objects). + private static final class Capture extends DisposableNativeObject implements Backend { + public native void initialize(); + public native void pump(CustomAudioSource source); + public native void dispose(); + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java new file mode 100644 index 000000000..3eec146a2 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import dev.onvoid.webrtc.TestBase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import static org.junit.jupiter.api.Assertions.*; + +/** Opt in with -Dtest.systemAudio=true while playing audio on the default output. */ +@EnabledIfSystemProperty(named = "test.systemAudio", matches = "true") +class SystemAudioSourceIntegrationTest extends TestBase { + @Test + void capturesOutputAndRestarts() throws Exception { + try (SystemAudioSource source = new SystemAudioSource()) { + AudioTrack track = factory.createAudioTrack("system", source); + AtomicReference received = new AtomicReference<>(new CountDownLatch(5)); + try { + track.addSink((data, bits, rate, channels, frames) -> { + if (bits == 16 && rate == 48000 && channels == 2 && frames == 480) { + for (byte sample : data) { + if (sample != 0) { received.get().countDown(); break; } + } + } + }); + source.start(); + assertTrue(received.get().await(10, TimeUnit.SECONDS), "Play audio on the default output"); + source.stop(); + assertNull(source.getFailure()); + assertFalse(source.isRecording()); + received.set(new CountDownLatch(5)); + source.start(); + assertTrue(received.get().await(10, TimeUnit.SECONDS), "No audio after restart"); + assertTrue(source.isRecording()); + source.stop(); + assertNull(source.getFailure()); + } + finally { track.dispose(); } + } + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java new file mode 100644 index 000000000..5e52424fe --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java @@ -0,0 +1,195 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import dev.onvoid.webrtc.TestBase; +import org.junit.jupiter.api.Test; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import static org.junit.jupiter.api.Assertions.*; + +class SystemAudioSourceTest extends TestBase { + private static class Backend implements SystemAudioSource.Backend { + final CountDownLatch pumped = new CountDownLatch(1); + final CountDownLatch closed = new CountDownLatch(1); + Thread thread; + public void initialize() { thread = Thread.currentThread(); } + public void pump(CustomAudioSource source) { + assertSame(thread, Thread.currentThread()); + pumped.countDown(); + } + public void dispose() { + assertSame(thread, Thread.currentThread()); + closed.countDown(); + } + } + + @Test + void restartAndDispose() throws Exception { + AtomicInteger opened = new AtomicInteger(); + AtomicReference latest = new AtomicReference<>(); + SystemAudioSource source = new SystemAudioSource(() -> { + opened.incrementAndGet(); + Backend backend = new Backend(); + latest.set(backend); + return backend; + }); + try { + assertFalse(source.isRecording()); + source.stop(); + source.start(); + assertTrue(latest.get().pumped.await(5, TimeUnit.SECONDS)); + source.start(); + assertEquals(1, opened.get()); + source.stop(); + assertEquals(0, latest.get().closed.getCount()); + assertFalse(source.isRecording()); + source.start(); + assertEquals(2, opened.get()); + assertNull(source.getFailure()); + } + finally { source.dispose(); } + source.dispose(); + assertThrows(IllegalStateException.class, source::start); + assertThrows(IllegalStateException.class, + () -> source.pushAudio(new byte[1920], 16, 48000, 2, 480)); + } + + @Test + void startupFailureClosesBackend() { + IllegalStateException denied = new IllegalStateException("permission denied"); + Backend backend = new Backend() { + public void initialize() { super.initialize(); throw denied; } + }; + try (SystemAudioSource source = new SystemAudioSource(() -> backend)) { + assertSame(denied, assertThrows(IllegalStateException.class, source::start).getCause()); + assertSame(denied, source.getFailure()); + assertEquals(0, backend.closed.getCount()); + assertFalse(source.isRecording()); + } + } + + @Test + void runtimeFailureCanRestart() throws Exception { + IllegalStateException lost = new IllegalStateException("output lost"); + Backend broken = new Backend() { + public void pump(CustomAudioSource source) { throw lost; } + }; + AtomicInteger attempts = new AtomicInteger(); + try (SystemAudioSource source = new SystemAudioSource( + () -> attempts.getAndIncrement() == 0 ? broken : new Backend())) { + source.start(); + assertTrue(broken.closed.await(5, TimeUnit.SECONDS)); + source.stop(); // Join also waits for publication of the failure. + assertSame(lost, source.getFailure()); + source.start(); + assertNull(source.getFailure()); + assertTrue(source.isRecording()); + } + } + + @Test + void callbackCanDisposeSource() throws Exception { + AtomicReference ref = new AtomicReference<>(); + Backend backend = new Backend() { + public void pump(CustomAudioSource source) { + ref.get().dispose(); + super.pump(source); + } + }; + SystemAudioSource source = new SystemAudioSource(() -> backend); + ref.set(source); + try { + source.start(); + assertTrue(backend.closed.await(5, TimeUnit.SECONDS)); + source.stop(); + assertThrows(IllegalStateException.class, source::start); + } + finally { source.dispose(); } + } + + @Test + void disposeDuringNativeCallback() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch disposing = new CountDownLatch(1); + Backend backend = new Backend() { + public void pump(CustomAudioSource source) { + source.pushAudio(new byte[1920], 16, 48000, 2, 480); + } + }; + SystemAudioSource source = new SystemAudioSource(() -> backend); + AudioTrack track = factory.createAudioTrack("system-callback", source); + CompletableFuture disposed = new CompletableFuture<>(); + try { + track.addSink((data, bits, rate, channels, frames) -> { + entered.countDown(); + try { + if (!disposing.await(5, TimeUnit.SECONDS)) return; + } + catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } + source.dispose(); + }); + source.start(); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + Thread closer = new Thread(() -> { + // Hold the lifecycle monitor before allowing the native sink + // callback to request disposal on the capture worker as well. + synchronized (source) { + disposing.countDown(); + try { source.dispose(); disposed.complete(null); } + catch (Throwable e) { disposed.completeExceptionally(e); } + } + }); + closer.setDaemon(true); + closer.start(); + disposed.get(5, TimeUnit.SECONDS); + assertFalse(source.isRecording()); + assertNull(source.getFailure()); + } + finally { + disposing.countDown(); + source.dispose(); + track.dispose(); + } + } + + @Test + void sourceFeedsNativeTrack() throws Exception { + CountDownLatch received = new CountDownLatch(1); + Backend backend = new Backend() { + public void pump(CustomAudioSource source) { + source.pushAudio(new byte[1920], 16, 48000, 2, 480); + } + }; + try (SystemAudioSource source = new SystemAudioSource(() -> backend)) { + AudioTrack track = factory.createAudioTrack("system", source); + try { + track.addSink((data, bits, rate, channels, frames) -> { + if (bits == 16 && rate == 48000 && channels == 2 && frames == 480) + received.countDown(); + }); + source.start(); + assertTrue(received.await(5, TimeUnit.SECONDS)); + source.stop(); + } + finally { track.dispose(); } + } + } +} From 15a4bf74bb0d6458d7ee933a93cbbe340ca4d6a7 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 8 Nov 2026 13:38:03 +0100 Subject: [PATCH 2/3] refactor: align system audio capture with project style Fix PulseAudio capture timestamps (enable timing updates, date fragments of one poll contiguously), tolerate packet jitter of one chunk before treating a gap as a pause, and reject pushAudio on SystemAudioSource from threads other than the capture worker. --- docs/guide/audio/system-audio-source.md | 3 +- .../webrtc/examples/SystemAudioExample.java | 37 +- .../main/cpp/include/JNI_SystemAudioSource.h | 54 +- .../include/media/audio/SystemAudioCapture.h | 84 ++- .../media/audio/linux/PulseAudioLoader.h | 64 ++- .../main/cpp/src/JNI_SystemAudioSource.cpp | 58 +- .../src/media/audio/SystemAudioCapture.cpp | 163 ++++-- .../media/audio/linux/SystemAudioCapture.cpp | 459 +++++++++++----- .../media/audio/macos/SystemAudioCapture.cpp | 455 ++++++++++------ .../audio/windows/SystemAudioCapture.cpp | 226 +++++--- .../webrtc/media/audio/SystemAudioSource.java | 496 ++++++++++++------ .../SystemAudioSourceIntegrationTest.java | 33 +- .../media/audio/SystemAudioSourceTest.java | 172 ++++-- 13 files changed, 1586 insertions(+), 718 deletions(-) diff --git a/docs/guide/audio/system-audio-source.md b/docs/guide/audio/system-audio-source.md index c4184f3a6..fac7ab893 100644 --- a/docs/guide/audio/system-audio-source.md +++ b/docs/guide/audio/system-audio-source.md @@ -56,7 +56,8 @@ Use `try/finally` in application code to release these resources on errors too. `stop()` and `dispose()` are idempotent. `stop()` keeps the source and its tracks usable for `start()` again; disposal is permanent. A stop or disposal from a local audio sink requests shutdown without joining its own callback thread. Do not -restart from an audio callback or push additional PCM into this source. +restart from an audio callback. The inherited `pushAudio` methods throw +`IllegalStateException`: push other audio into a separate `CustomAudioSource`. ## Device and permission changes diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java index c7c5ef9b2..e0ea8e01f 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java @@ -16,35 +16,59 @@ package dev.onvoid.webrtc.examples; +import java.util.concurrent.atomic.AtomicLong; + import dev.onvoid.webrtc.PeerConnectionFactory; import dev.onvoid.webrtc.media.audio.AudioDeviceModule; import dev.onvoid.webrtc.media.audio.AudioLayer; import dev.onvoid.webrtc.media.audio.AudioTrack; import dev.onvoid.webrtc.media.audio.SystemAudioSource; -import java.util.concurrent.atomic.AtomicLong; /** - * Captures the default system output into a native audio track without playing - * it back. Play audio in another application and press Enter to stop. - * For sending, add this track beside the desktop video track on a peer connection. + * Example demonstrating how to capture the audio the system plays, as a local + * audio track. + *

+ * This example shows how to: + *

    + *
  • Capture the default output with a {@link SystemAudioSource}
  • + *
  • Read the captured audio through a sink of its track
  • + *
  • Stop the capture and check it for a failure
  • + *
+ *

+ * The captured audio is counted, not played back. To send it, add the track to + * a peer connection, beside the track of a desktop video source. + *

+ * Play audio in another application, run this example, and press Enter to stop: + *

+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.SystemAudioExample"
+ * 
+ * + * @author Alex Andres */ public class SystemAudioExample { + public static void main(String[] args) throws Exception { - // SystemAudioSource opens its own output capture; no microphone is needed. + // The source opens its own output capture, so no microphone is needed. try (AudioDeviceModule audio = new AudioDeviceModule(AudioLayer.kDummyAudio); PeerConnectionFactory factory = new PeerConnectionFactory(audio); SystemAudioSource source = new SystemAudioSource()) { AudioTrack track = factory.createAudioTrack("system-audio", source); AtomicLong frames = new AtomicLong(); + try { track.addSink((data, bits, rate, channels, count) -> frames.addAndGet(count)); + source.start(); - System.out.println("Capturing system audio. Play audio in another app; press Enter to stop."); + + System.out.println("Capturing system audio. Play audio in another application; press Enter to stop."); System.in.read(); + source.stop(); + if (source.getFailure() != null) { throw new IllegalStateException("System audio capture failed", source.getFailure()); } + System.out.println("Captured " + frames.get() + " stereo PCM frames"); } finally { @@ -53,4 +77,5 @@ public static void main(String[] args) throws Exception { } } } + } diff --git a/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h b/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h index 7f143907f..6e81ab834 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_SystemAudioSource.h @@ -1,25 +1,37 @@ -/* - * Copyright 2026 Alex Andres - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef JNI_SYSTEM_AUDIO_SOURCE_H -#define JNI_SYSTEM_AUDIO_SOURCE_H +/* DO NOT EDIT THIS FILE - it is machine generated */ #include +/* Header for class dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture */ + +#ifndef _Included_dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture +#define _Included_dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture +#ifdef __cplusplus extern "C" { -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_initialize(JNIEnv *, jobject); -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_pump(JNIEnv *, jobject, jobject); -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_dispose(JNIEnv *, jobject); +#endif + /* + * Class: dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture + * Method: initialize + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_initialize + (JNIEnv *, jobject); + + /* + * Class: dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture + * Method: pump + * Signature: (Ldev/onvoid/webrtc/media/audio/CustomAudioSource;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_pump + (JNIEnv *, jobject, jobject); + + /* + * Class: dev_onvoid_webrtc_media_audio_SystemAudioSource_Capture + * Method: dispose + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_dispose + (JNIEnv *, jobject); + +#ifdef __cplusplus } #endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h b/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h index 87686fbd9..ed4d4c8c8 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/SystemAudioCapture.h @@ -14,37 +14,73 @@ * limitations under the License. */ -#ifndef JNI_SYSTEM_AUDIO_CAPTURE_H -#define JNI_SYSTEM_AUDIO_CAPTURE_H +#ifndef JNI_WEBRTC_MEDIA_AUDIO_SYSTEM_AUDIO_CAPTURE_H_ +#define JNI_WEBRTC_MEDIA_AUDIO_SYSTEM_AUDIO_CAPTURE_H_ #include "media/audio/CustomAudioSource.h" + #include +#include +#include #include #include #include #include -namespace jni { -// Platform callbacks only write PCM into a bounded queue; they never call Java -// or WebRTC. The owning Java worker polls and delivers complete 10 ms chunks. -class SystemAudioCapture { -public: - virtual ~SystemAudioCapture() = default; - virtual void Open() = 0; - virtual void Poll() = 0; - void Pump(CustomAudioSource * source); - void Append(const int16_t * samples, size_t frames, int64_t timestamp_us); - void Fail(const char * message) noexcept; - static std::unique_ptr Create(); -private: - struct Chunk { - std::array samples{}; - size_t frames = 0; - int64_t time = 0; - }; - std::mutex mutex_; - std::deque chunks_; - std::string failure_; -}; +namespace jni +{ + // Captures what the system plays on its default output, as 16-bit 48 kHz + // stereo PCM. Each platform implements Open() and Poll(). + // + // Platform callbacks only write PCM into a bounded queue; they never call + // Java or WebRTC. The owning Java worker calls Pump(), which polls the + // platform and delivers the complete 10 ms chunks to a CustomAudioSource. + // Open(), Pump() and the destructor run on that one worker thread. + class SystemAudioCapture + { + public: + static constexpr int kSampleRate = 48000; + static constexpr size_t kChannels = 2; + static constexpr size_t kFramesPerChunk = kSampleRate / 100; + + // Creates the capture of the current platform, not opened yet. + static std::unique_ptr Create(); + + virtual ~SystemAudioCapture() = default; + + // Opens the default output. Throws an Exception on failure. + virtual void Open() = 0; + + // Appends the audio the platform has buffered since the last call. + // Throws an Exception once capture has failed. + virtual void Poll() = 0; + + // Polls the platform and pushes every complete chunk to the source, + // stamped with its capture time. + void Pump(CustomAudioSource * source); + + // Queues interleaved frames whose first one was captured at the given + // time, on the clock of webrtc::TimeMicros(). Null samples queue + // silence of the given length. + void Append(const int16_t * samples, size_t frames, int64_t timestamp_us); + + // Records a failure, which the next Pump() throws. Safe to call from + // platform callbacks. + void Fail(const char * message) noexcept; + + private: + struct Chunk + { + std::array samples{}; + size_t frames = 0; + int64_t timestamp_us = 0; + }; + + // Guards chunks_ and failure_. + std::mutex mutex_; + std::deque chunks_; + std::string failure_; + }; } + #endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h index cdfdd0a57..91ffa70b8 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioLoader.h @@ -21,8 +21,8 @@ #include -#include #include +#include #include "Exception.h" @@ -61,6 +61,16 @@ namespace jni typedef pa_operation_state_t (*pa_operation_get_state_t)(const pa_operation* o); typedef const char* (*pa_proplist_gets_t)(const pa_proplist* p, const char* key); + typedef pa_stream* (*pa_stream_new_t)(pa_context* c, const char* name, const pa_sample_spec* ss, const pa_channel_map* map); + typedef void (*pa_stream_unref_t)(pa_stream* s); + typedef int (*pa_stream_connect_record_t)(pa_stream* s, const char* dev, const pa_buffer_attr* attr, pa_stream_flags_t flags); + typedef int (*pa_stream_disconnect_t)(pa_stream* s); + typedef pa_stream_state_t (*pa_stream_get_state_t)(const pa_stream* s); + typedef int (*pa_stream_peek_t)(pa_stream* s, const void** data, size_t* nbytes); + typedef int (*pa_stream_drop_t)(pa_stream* s); + typedef size_t (*pa_stream_readable_size_t)(const pa_stream* s); + typedef int (*pa_stream_get_latency_t)(pa_stream* s, pa_usec_t* r_usec, int* negative); + class PulseAudioLoader { public: static PulseAudioLoader& instance() { @@ -112,24 +122,16 @@ namespace jni LOAD_SYM(pa_operation_get_state); LOAD_SYM(pa_proplist_gets); - pa_stream_new = reinterpret_cast(dlsym(lib_handle, "pa_stream_new")); - if (!pa_stream_new) { close(); return false; } - pa_stream_unref = reinterpret_cast(dlsym(lib_handle, "pa_stream_unref")); - if (!pa_stream_unref) { close(); return false; } - pa_stream_connect_record = reinterpret_cast(dlsym(lib_handle, "pa_stream_connect_record")); - if (!pa_stream_connect_record) { close(); return false; } - pa_stream_disconnect = reinterpret_cast(dlsym(lib_handle, "pa_stream_disconnect")); - if (!pa_stream_disconnect) { close(); return false; } - pa_stream_get_state = reinterpret_cast(dlsym(lib_handle, "pa_stream_get_state")); - if (!pa_stream_get_state) { close(); return false; } - pa_stream_peek = reinterpret_cast(dlsym(lib_handle, "pa_stream_peek")); - if (!pa_stream_peek) { close(); return false; } - pa_stream_drop = reinterpret_cast(dlsym(lib_handle, "pa_stream_drop")); - if (!pa_stream_drop) { close(); return false; } - pa_stream_readable_size = reinterpret_cast(dlsym(lib_handle, "pa_stream_readable_size")); - if (!pa_stream_readable_size) { close(); return false; } - pa_stream_get_latency = reinterpret_cast(dlsym(lib_handle, "pa_stream_get_latency")); - if (!pa_stream_get_latency) { close(); return false; } + LOAD_SYM(pa_stream_new); + LOAD_SYM(pa_stream_unref); + LOAD_SYM(pa_stream_connect_record); + LOAD_SYM(pa_stream_disconnect); + LOAD_SYM(pa_stream_get_state); + LOAD_SYM(pa_stream_peek); + LOAD_SYM(pa_stream_drop); + LOAD_SYM(pa_stream_readable_size); + LOAD_SYM(pa_stream_get_latency); + loaded = true; return true; } @@ -176,19 +178,23 @@ namespace jni pa_operation_get_state_t pa_operation_get_state; pa_proplist_gets_t pa_proplist_gets; - decltype(&::pa_stream_new) pa_stream_new = nullptr; - decltype(&::pa_stream_unref) pa_stream_unref = nullptr; - decltype(&::pa_stream_connect_record) pa_stream_connect_record = nullptr; - decltype(&::pa_stream_disconnect) pa_stream_disconnect = nullptr; - decltype(&::pa_stream_get_state) pa_stream_get_state = nullptr; - decltype(&::pa_stream_peek) pa_stream_peek = nullptr; - decltype(&::pa_stream_drop) pa_stream_drop = nullptr; - decltype(&::pa_stream_readable_size) pa_stream_readable_size = nullptr; - decltype(&::pa_stream_get_latency) pa_stream_get_latency = nullptr; + pa_stream_new_t pa_stream_new; + pa_stream_unref_t pa_stream_unref; + pa_stream_connect_record_t pa_stream_connect_record; + pa_stream_disconnect_t pa_stream_disconnect; + pa_stream_get_state_t pa_stream_get_state; + pa_stream_peek_t pa_stream_peek; + pa_stream_drop_t pa_stream_drop; + pa_stream_readable_size_t pa_stream_readable_size; + pa_stream_get_latency_t pa_stream_get_latency; private: - std::mutex load_mutex; PulseAudioLoader() : loaded(false), lib_handle(nullptr) {} + + // Serializes load(): the device manager and the system audio + // capture may load the library from different threads. + std::mutex load_mutex; + bool loaded; void* lib_handle; }; diff --git a/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp index b4b32d028..1b6461477 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_SystemAudioSource.cpp @@ -17,31 +17,49 @@ #include "JNI_SystemAudioSource.h" #include "JavaBoundary.h" #include "NativeHandle.h" + +#include "media/audio/CustomAudioSource.h" #include "media/audio/SystemAudioCapture.h" +#include + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_initialize -(JNIEnv * env, jobject caller) { - jni::CallFromJava(env, [&]() { - auto capture = jni::SystemAudioCapture::Create(); - capture->Open(); - SetHandle(env, caller, capture.get()); - capture.release(); - }); +(JNIEnv * env, jobject caller) +{ + jni::CallFromJava(env, [&]() { + std::unique_ptr capture = jni::SystemAudioCapture::Create(); + capture->Open(); + + SetHandle(env, caller, capture.release()); + }); } + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_pump -(JNIEnv * env, jobject caller, jobject javaSource) { - jni::CallFromJava(env, [&]() { - jni::NativeLease capture(env, caller); - CHECK_HANDLE(capture); - jni::NativeLease source(env, javaSource); - CHECK_HANDLE(source); - capture->Pump(source); - }); +(JNIEnv * env, jobject caller, jobject javaSource) +{ + jni::CallFromJava(env, [&]() { + jni::NativeLease capture(env, caller); + CHECK_HANDLE(capture); + + jni::NativeLease source(env, javaSource); + CHECK_HANDLE(source); + + capture->Pump(source); + }); } + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_SystemAudioSource_00024Capture_dispose -(JNIEnv * env, jobject caller) { - jni::CallFromJava(env, [&]() { - auto * capture = jni::TakeHandle(env, caller); - if (capture) jni::ReleaseHandle(env, capture, [capture](bool) { delete capture; }); - }); +(JNIEnv * env, jobject caller) +{ + jni::CallFromJava(env, [&]() { + jni::SystemAudioCapture * capture = jni::TakeHandle(env, caller); + + if (capture == nullptr) { + return; + } + + jni::ReleaseHandle(env, capture, [capture](bool) { + delete capture; + }); + }); } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp index 7bc58d670..50f363e5b 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/SystemAudioCapture.cpp @@ -15,63 +15,116 @@ */ #include "media/audio/SystemAudioCapture.h" + #include "Exception.h" + +#include "rtc_base/time_utils.h" + #include +#include #include +#include -namespace jni { -void SystemAudioCapture::Append(const int16_t * samples, size_t frames, int64_t time) { - std::lock_guard lock(mutex_); - // Do not splice a partial chunk from before an output pause into new - // audio. The gap must remain a timestamp gap on the outgoing stream. - if (!chunks_.empty() && chunks_.back().frames < 480) { - const auto & tail = chunks_.back(); - const int64_t expected = tail.time + tail.frames * 1000000 / 48000; - if (time - expected > 5000 || expected - time > 5000) chunks_.pop_back(); - } - size_t offset = 0; - while (offset < frames) { - if (chunks_.empty() || chunks_.back().frames == 480) { - // Bound latency and memory if the consumer stalls. Preserve capture - // timestamps when dropping old audio instead of replaying a backlog. - if (chunks_.size() >= 10) chunks_.pop_front(); - chunks_.emplace_back(); - chunks_.back().time = time + offset * 1000000 / 48000; - } - auto & chunk = chunks_.back(); - const size_t count = std::min(frames - offset, 480 - chunk.frames); - if (samples) { - std::memcpy(chunk.samples.data() + chunk.frames * 2, - samples + offset * 2, count * 4); - } - else { - std::memset(chunk.samples.data() + chunk.frames * 2, 0, count * 4); - } - chunk.frames += count; - offset += count; - } -} -void SystemAudioCapture::Fail(const char * message) noexcept { - // Native OS callbacks must never unwind through platform code. - try { - std::lock_guard lock(mutex_); - failure_ = message; - } - catch (...) { } -} -void SystemAudioCapture::Pump(CustomAudioSource * source) { - Poll(); - std::deque ready; - { - std::lock_guard lock(mutex_); - if (!failure_.empty()) throw Exception(failure_.c_str()); - while (!chunks_.empty() && chunks_.front().frames == 480) { - ready.push_back(std::move(chunks_.front())); - chunks_.pop_front(); - } - } - for (auto & chunk : ready) { - source->PushAudioData(chunk.samples.data(), 16, 48000, 2, 480, chunk.time); - } -} +namespace jni +{ + namespace + { + // The most chunks queued, 100 ms. When the worker falls behind, the + // oldest audio is dropped rather than sent late as a backlog. + constexpr size_t kMaxQueuedChunks = 10; + + // How far a packet may start from where the queued audio ends and still + // continue it. Packet timestamps jitter by a few milliseconds; a pause + // of the output is much longer. + constexpr int64_t kMaxTimestampJitterUs = 10 * webrtc::kNumMicrosecsPerMillisec; + + int64_t FramesToMicros(size_t frames) + { + return static_cast(frames) * webrtc::kNumMicrosecsPerSec / SystemAudioCapture::kSampleRate; + } + } + + void SystemAudioCapture::Append(const int16_t * samples, size_t frames, int64_t timestamp_us) + { + std::lock_guard lock(mutex_); + + // Do not splice a partial chunk from before an output pause into new + // audio. The pause must remain a timestamp gap on the outgoing stream. + if (!chunks_.empty() && chunks_.back().frames < kFramesPerChunk) { + const Chunk & tail = chunks_.back(); + const int64_t expected_us = tail.timestamp_us + FramesToMicros(tail.frames); + + if (std::abs(timestamp_us - expected_us) > kMaxTimestampJitterUs) { + chunks_.pop_back(); + } + } + + size_t offset = 0; + + while (offset < frames) { + if (chunks_.empty() || chunks_.back().frames == kFramesPerChunk) { + // Bound latency and memory if the consumer stalls. Dropping old + // chunks keeps the capture time of the remaining ones. + if (chunks_.size() >= kMaxQueuedChunks) { + chunks_.pop_front(); + } + + chunks_.emplace_back(); + chunks_.back().timestamp_us = timestamp_us + FramesToMicros(offset); + } + + Chunk & chunk = chunks_.back(); + const size_t count = std::min(frames - offset, kFramesPerChunk - chunk.frames); + int16_t * target = chunk.samples.data() + chunk.frames * kChannels; + + if (samples != nullptr) { + std::memcpy(target, samples + offset * kChannels, count * kChannels * sizeof(int16_t)); + } + else { + std::memset(target, 0, count * kChannels * sizeof(int16_t)); + } + + chunk.frames += count; + offset += count; + } + } + + void SystemAudioCapture::Fail(const char * message) noexcept + { + // Native OS callbacks must never unwind through platform code. + try { + std::lock_guard lock(mutex_); + + failure_ = message; + } + catch (...) { + } + } + + void SystemAudioCapture::Pump(CustomAudioSource * source) + { + Poll(); + + std::deque ready; + + { + std::lock_guard lock(mutex_); + + if (!failure_.empty()) { + throw Exception(failure_.c_str()); + } + + while (!chunks_.empty() && chunks_.front().frames == kFramesPerChunk) { + ready.push_back(std::move(chunks_.front())); + chunks_.pop_front(); + } + } + + // Deliver without the lock: the sinks may take long, and the platform + // callbacks must not wait for them. + for (const Chunk & chunk : ready) { + source->PushAudioData(chunk.samples.data(), 16, kSampleRate, kChannels, + kFramesPerChunk, chunk.timestamp_us); + } + } } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp index 0b8e695c4..78718449e 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/linux/SystemAudioCapture.cpp @@ -16,136 +16,341 @@ #include "media/audio/SystemAudioCapture.h" #include "media/audio/linux/PulseAudioLoader.h" + #include "Exception.h" + #include "rtc_base/time_utils.h" + #include +#include #include -namespace jni { -namespace { -class PulseCapture final : public SystemAudioCapture { - avdev::PulseAudioLoader & pa = avdev::PulseAudioLoader::instance(); - pa_threaded_mainloop * loop = nullptr; - pa_context * context = nullptr; - pa_stream * stream = nullptr; - bool started = false; - std::string sink, monitor; - bool infoDone = false; - struct Lock { - PulseCapture & self; - explicit Lock(PulseCapture & s) : self(s) { self.pa.pa_threaded_mainloop_lock(self.loop); } - ~Lock() { self.pa.pa_threaded_mainloop_unlock(self.loop); } - }; - template void Await(Test test) { - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); - for (;;) { - { - Lock lock(*this); - if (!PA_CONTEXT_IS_GOOD(pa.pa_context_get_state(context))) - throw Exception("System audio: PulseAudio connection lost"); - if (test()) return; - } - if (std::chrono::steady_clock::now() >= deadline) - throw Exception("System audio: PulseAudio startup timed out"); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - } - } -public: - ~PulseCapture() override { - if (loop && started) { - { - Lock lock(*this); - if (stream) { pa.pa_stream_disconnect(stream); pa.pa_stream_unref(stream); } - if (context) { pa.pa_context_disconnect(context); pa.pa_context_unref(context); } - } - pa.pa_threaded_mainloop_stop(loop); - } - if (loop) pa.pa_threaded_mainloop_free(loop); - } - void Open() override { - if (!pa.load()) throw Exception("System audio requires libpulse.so.0"); - loop = pa.pa_threaded_mainloop_new(); - if (!loop) throw Exception("System audio: cannot create PulseAudio mainloop"); - if (pa.pa_threaded_mainloop_start(loop) < 0) - throw Exception("System audio: cannot start PulseAudio mainloop"); - started = true; - { - Lock lock(*this); - context = pa.pa_context_new(pa.pa_threaded_mainloop_get_api(loop), "webrtc-java system audio"); - if (!context || pa.pa_context_connect(context, nullptr, PA_CONTEXT_NOAUTOSPAWN, nullptr) < 0) - throw Exception("System audio: cannot connect to PulseAudio / pipewire-pulse"); - } - Await([&] { return pa.pa_context_get_state(context) == PA_CONTEXT_READY; }); - { - Lock lock(*this); - auto * op = pa.pa_context_get_server_info(context, - [](pa_context *, const pa_server_info * info, void * data) noexcept { - auto * self = static_cast(data); - try { if (info && info->default_sink_name) self->sink = info->default_sink_name; } - catch (...) { self->Fail("System audio: cannot read default sink"); } - self->infoDone = true; - }, this); - if (!op) throw Exception("System audio: cannot query default output"); - pa.pa_operation_unref(op); - } - Await([&] { return infoDone; }); - { - Lock lock(*this); - if (sink.empty()) throw Exception("System audio: no default output"); - infoDone = false; - auto * op = pa.pa_context_get_sink_info_by_name(context, sink.c_str(), - [](pa_context *, const pa_sink_info * info, int end, void * data) noexcept { - auto * self = static_cast(data); - try { if (info && info->monitor_source_name) self->monitor = info->monitor_source_name; } - catch (...) { self->Fail("System audio: cannot read output monitor"); } - if (end) self->infoDone = true; - }, this); - if (!op) throw Exception("System audio: cannot query output monitor"); - pa.pa_operation_unref(op); - } - Await([&] { return infoDone; }); - { - Lock lock(*this); - if (monitor.empty()) throw Exception("System audio: output has no monitor source"); - pa_sample_spec spec{PA_SAMPLE_S16NE, 48000, 2}; - stream = pa.pa_stream_new(context, "System output", &spec, nullptr); - if (!stream) throw Exception("System audio: cannot create monitor stream"); - pa_buffer_attr attr{19200, UINT32_MAX, UINT32_MAX, UINT32_MAX, 1920}; - if (pa.pa_stream_connect_record(stream, monitor.c_str(), &attr, - static_cast(PA_STREAM_ADJUST_LATENCY | PA_STREAM_DONT_MOVE)) < 0) - throw Exception("System audio: cannot open output monitor"); - } - Await([&] { - const auto state = pa.pa_stream_get_state(stream); - if (!PA_STREAM_IS_GOOD(state)) throw Exception("System audio: monitor unavailable or permission denied"); - return state == PA_STREAM_READY; - }); - } - void Poll() override { - Lock lock(*this); - if (pa.pa_context_get_state(context) != PA_CONTEXT_READY || pa.pa_stream_get_state(stream) != PA_STREAM_READY) - throw Exception("System audio: output lost; stop and restart after selecting an output"); - // A bounded drain keeps stop responsive even if an output is very busy. - for (int i = 0; i < 10; ++i) { - size_t available = pa.pa_stream_readable_size(stream); - if (available == static_cast(-1)) throw Exception("System audio: monitor read failed"); - if (!available) break; - const void * data = nullptr; - size_t bytes = 0; - if (pa.pa_stream_peek(stream, &data, &bytes) < 0) throw Exception("System audio: monitor peek failed"); - if (!bytes) break; - pa_usec_t latency = 0; - int negative = 0; - if (pa.pa_stream_get_latency(stream, &latency, &negative) < 0 || negative) latency = 0; - try { - Append(static_cast(data), bytes / 4, - webrtc::TimeMicros() - static_cast(latency)); - } - catch (...) { pa.pa_stream_drop(stream); throw; } - if (pa.pa_stream_drop(stream) < 0) throw Exception("System audio: monitor drop failed"); - } - } -}; -} -std::unique_ptr SystemAudioCapture::Create() { return std::make_unique(); } +namespace jni +{ + namespace + { + // The bytes of one 16-bit stereo frame. + constexpr size_t kFrameBytes = SystemAudioCapture::kChannels * sizeof(int16_t); + + // How long each step of opening the monitor may take. + constexpr std::chrono::seconds kStartupTimeout(10); + + // The most fragments read by one Poll(), so that stopping stays + // responsive even while an output is very busy. + constexpr int kMaxFragmentsPerPoll = 10; + + // Records the monitor source of the default PulseAudio sink, which + // pipewire-pulse provides as well. + class PulseAudioCapture final : public SystemAudioCapture + { + public: + ~PulseAudioCapture() override; + + void Open() override; + void Poll() override; + + private: + // Holds the lock of the threaded mainloop for its scope. + class MainloopLock + { + public: + explicit MainloopLock(PulseAudioCapture & capture) : + capture_(capture) + { + capture_.pa_.pa_threaded_mainloop_lock(capture_.mainloop_); + } + + ~MainloopLock() + { + capture_.pa_.pa_threaded_mainloop_unlock(capture_.mainloop_); + } + + private: + PulseAudioCapture & capture_; + }; + + // Polls the condition under the mainloop lock until it holds, + // the context fails, or the startup timeout expires. + template + void Await(Condition condition); + + avdev::PulseAudioLoader & pa_ = avdev::PulseAudioLoader::instance(); + pa_threaded_mainloop * mainloop_ = nullptr; + pa_context * context_ = nullptr; + pa_stream * stream_ = nullptr; + bool mainloopStarted_ = false; + + // Written by the mainloop callbacks and read by Open(), both + // under the mainloop lock. + std::string sinkName_; + std::string monitorName_; + bool infoDone_ = false; + }; + + PulseAudioCapture::~PulseAudioCapture() + { + if (mainloop_ != nullptr && mainloopStarted_) { + { + MainloopLock lock(*this); + + if (stream_ != nullptr) { + pa_.pa_stream_disconnect(stream_); + pa_.pa_stream_unref(stream_); + } + if (context_ != nullptr) { + pa_.pa_context_disconnect(context_); + pa_.pa_context_unref(context_); + } + } + + pa_.pa_threaded_mainloop_stop(mainloop_); + } + if (mainloop_ != nullptr) { + pa_.pa_threaded_mainloop_free(mainloop_); + } + } + + template + void PulseAudioCapture::Await(Condition condition) + { + const auto deadline = std::chrono::steady_clock::now() + kStartupTimeout; + + while (true) { + { + MainloopLock lock(*this); + + if (!PA_CONTEXT_IS_GOOD(pa_.pa_context_get_state(context_))) { + throw Exception("System audio: PulseAudio connection lost"); + } + if (condition()) { + return; + } + } + + if (std::chrono::steady_clock::now() >= deadline) { + throw Exception("System audio: PulseAudio startup timed out"); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + } + + void PulseAudioCapture::Open() + { + if (!pa_.load()) { + throw Exception("System audio requires libpulse.so.0"); + } + + mainloop_ = pa_.pa_threaded_mainloop_new(); + + if (mainloop_ == nullptr) { + throw Exception("System audio: Cannot create PulseAudio mainloop"); + } + if (pa_.pa_threaded_mainloop_start(mainloop_) < 0) { + throw Exception("System audio: Cannot start PulseAudio mainloop"); + } + + mainloopStarted_ = true; + + { + MainloopLock lock(*this); + + context_ = pa_.pa_context_new(pa_.pa_threaded_mainloop_get_api(mainloop_), "webrtc-java system audio"); + + if (context_ == nullptr || pa_.pa_context_connect(context_, nullptr, PA_CONTEXT_NOAUTOSPAWN, nullptr) < 0) { + throw Exception("System audio: Cannot connect to PulseAudio or pipewire-pulse"); + } + } + + Await([this]() { + return pa_.pa_context_get_state(context_) == PA_CONTEXT_READY; + }); + + { + MainloopLock lock(*this); + + pa_operation * op = pa_.pa_context_get_server_info(context_, + [](pa_context *, const pa_server_info * info, void * data) noexcept { + auto * self = static_cast(data); + + try { + if (info != nullptr && info->default_sink_name != nullptr) { + self->sinkName_ = info->default_sink_name; + } + } + catch (...) { + self->Fail("System audio: Cannot read the default output"); + } + + self->infoDone_ = true; + }, this); + + if (op == nullptr) { + throw Exception("System audio: Cannot query the default output"); + } + + pa_.pa_operation_unref(op); + } + + Await([this]() { + return infoDone_; + }); + + { + MainloopLock lock(*this); + + if (sinkName_.empty()) { + throw Exception("System audio: No default output"); + } + + infoDone_ = false; + + pa_operation * op = pa_.pa_context_get_sink_info_by_name(context_, sinkName_.c_str(), + [](pa_context *, const pa_sink_info * info, int eol, void * data) noexcept { + auto * self = static_cast(data); + + try { + if (info != nullptr && info->monitor_source_name != nullptr) { + self->monitorName_ = info->monitor_source_name; + } + } + catch (...) { + self->Fail("System audio: Cannot read the output monitor"); + } + + // Also set on an error (eol < 0), so that Open() fails + // with no monitor rather than waiting for the timeout. + if (eol != 0) { + self->infoDone_ = true; + } + }, this); + + if (op == nullptr) { + throw Exception("System audio: Cannot query the output monitor"); + } + + pa_.pa_operation_unref(op); + } + + Await([this]() { + return infoDone_; + }); + + { + MainloopLock lock(*this); + + if (monitorName_.empty()) { + throw Exception("System audio: The output has no monitor source"); + } + + pa_sample_spec spec { PA_SAMPLE_S16NE, kSampleRate, static_cast(kChannels) }; + + stream_ = pa_.pa_stream_new(context_, "System output", &spec, nullptr); + + if (stream_ == nullptr) { + throw Exception("System audio: Cannot create the monitor stream"); + } + + // Buffer at most 100 ms, delivered in fragments of 10 ms. + pa_buffer_attr attr {}; + attr.maxlength = static_cast(10 * kFramesPerChunk * kFrameBytes); + attr.tlength = UINT32_MAX; + attr.prebuf = UINT32_MAX; + attr.minreq = UINT32_MAX; + attr.fragsize = static_cast(kFramesPerChunk * kFrameBytes); + + // The timing flags keep the stream's latency up to date, which + // dates the captured audio. Without them it is never known. + auto flags = static_cast(PA_STREAM_ADJUST_LATENCY | PA_STREAM_DONT_MOVE | + PA_STREAM_AUTO_TIMING_UPDATE | PA_STREAM_INTERPOLATE_TIMING); + + if (pa_.pa_stream_connect_record(stream_, monitorName_.c_str(), &attr, flags) < 0) { + throw Exception("System audio: Cannot open the output monitor"); + } + } + + Await([this]() { + const pa_stream_state_t state = pa_.pa_stream_get_state(stream_); + + if (!PA_STREAM_IS_GOOD(state)) { + throw Exception("System audio: Output monitor unavailable or permission denied"); + } + + return state == PA_STREAM_READY; + }); + } + + void PulseAudioCapture::Poll() + { + MainloopLock lock(*this); + + if (pa_.pa_context_get_state(context_) != PA_CONTEXT_READY || + pa_.pa_stream_get_state(stream_) != PA_STREAM_READY) { + throw Exception("System audio: Output lost; stop and restart after selecting an output"); + } + + // The fragments read by one call follow each other without a gap, so + // all of them are dated from the latency read for the first one. + // Dating each from a latency of its own would make them overlap. + int64_t firstCaptureUs = 0; + size_t framesRead = 0; + + for (int i = 0; i < kMaxFragmentsPerPoll; ++i) { + size_t available = pa_.pa_stream_readable_size(stream_); + + if (available == static_cast(-1)) { + throw Exception("System audio: Monitor read failed"); + } + if (available == 0) { + break; + } + + const void * data = nullptr; + size_t bytes = 0; + + if (pa_.pa_stream_peek(stream_, &data, &bytes) < 0) { + throw Exception("System audio: Monitor peek failed"); + } + if (bytes == 0) { + break; + } + + if (framesRead == 0) { + // The latency of a record stream is how long ago the next + // byte to read was captured. + pa_usec_t latency = 0; + int negative = 0; + + if (pa_.pa_stream_get_latency(stream_, &latency, &negative) < 0 || negative) { + latency = 0; + } + + firstCaptureUs = webrtc::TimeMicros() - static_cast(latency); + } + + const size_t frames = bytes / kFrameBytes; + const int64_t captureUs = firstCaptureUs + static_cast(framesRead) * webrtc::kNumMicrosecsPerSec / kSampleRate; + + try { + // A hole in the stream comes as null data, queued as silence. + Append(static_cast(data), frames, captureUs); + } + catch (...) { + pa_.pa_stream_drop(stream_); + throw; + } + + if (pa_.pa_stream_drop(stream_) < 0) { + throw Exception("System audio: Monitor drop failed"); + } + + framesRead += frames; + } + } + } + + std::unique_ptr SystemAudioCapture::Create() + { + return std::make_unique(); + } } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp index 06715dd4b..d49bd25df 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/macos/SystemAudioCapture.cpp @@ -15,177 +15,324 @@ */ #include "media/audio/SystemAudioCapture.h" + #include "Exception.h" + #include "rtc_base/time_utils.h" -#import + #import -#include +#import + #include #include -#include +#include #include +#include +#include +#include #include -// A receiver may outlive a timed-out start/stop. Detach its owner under the -// same lock used by callbacks before destroying the native capture object. +// Receives the audio of an SCStream. A receiver may outlive a timed-out start +// or stop, so the capture detaches itself as its owner, under the same lock +// the callbacks take, before the native capture object is destroyed. API_AVAILABLE(macos(13.0)) @interface SystemAudioReceiver : NSObject { @public - jni::SystemAudioCapture * owner; + jni::SystemAudioCapture * owner; } @end @implementation SystemAudioReceiver -- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { - @synchronized(self) { - if (owner) owner->Fail("System audio: ScreenCaptureKit stopped; check recording permission and restart"); - } + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error +{ + @synchronized(self) { + if (owner) { + owner->Fail("System audio: ScreenCaptureKit stopped; check recording permission and restart"); + } + } } -- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sample ofType:(SCStreamOutputType)type { - if (type != SCStreamOutputTypeAudio || !CMSampleBufferDataIsReady(sample)) return; - @synchronized(self) { - if (!owner) return; - try { - const auto * format = CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sample)); - if (!format || format->mFormatID != kAudioFormatLinearPCM || - format->mSampleRate != 48000 || format->mChannelsPerFrame != 2 || - !(format->mFormatFlags & kAudioFormatFlagIsFloat) || format->mBitsPerChannel != 32) { - owner->Fail("System audio: unexpected ScreenCaptureKit PCM format"); - return; - } - struct { AudioBufferList list; AudioBuffer extra; } storage{}; - CMBlockBufferRef block = nullptr; - OSStatus status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sample, nullptr, - &storage.list, sizeof(storage), kCFAllocatorDefault, kCFAllocatorDefault, - kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &block); - // Release the retained block even if allocating the conversion buffer fails. - std::unique_ptr retained(block, &CFRelease); - if (status != noErr) { owner->Fail("System audio: cannot read ScreenCaptureKit PCM"); return; } - const size_t frames = CMSampleBufferGetNumSamples(sample); - const bool planar = format->mFormatFlags & kAudioFormatFlagIsNonInterleaved; - const auto & list = storage.list; - if (list.mNumberBuffers != (planar ? 2U : 1U)) { - owner->Fail("System audio: unexpected ScreenCaptureKit channel layout"); return; - } - for (UInt32 b = 0; b < list.mNumberBuffers; ++b) { - if (!list.mBuffers[b].mData || list.mBuffers[b].mDataByteSize < frames * sizeof(float) * (planar ? 1 : 2)) { - owner->Fail("System audio: incomplete ScreenCaptureKit PCM"); return; - } - } - std::vector pcm(frames * 2); - for (size_t f = 0; f < frames; ++f) { - for (size_t c = 0; c < 2; ++c) { - float value = static_cast(list.mBuffers[planar ? c : 0].mData)[planar ? f : f * 2 + c]; - if (!std::isfinite(value)) value = 0; - pcm[f * 2 + c] = static_cast(std::max(-32768.0f, std::min(32767.0f, value * 32768.0f))); - } - } - const double pts = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sample)); - const double now = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())); - if (!std::isfinite(pts) || !std::isfinite(now)) { - owner->Fail("System audio: invalid ScreenCaptureKit timestamp"); return; - } - owner->Append(pcm.data(), frames, webrtc::TimeMicros() + static_cast((pts - now) * 1000000)); - } - catch (...) { owner->Fail("System audio: ScreenCaptureKit callback failed"); } - } + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sample ofType:(SCStreamOutputType)type +{ + if (type != SCStreamOutputTypeAudio || !CMSampleBufferDataIsReady(sample)) { + return; + } + + @synchronized(self) { + if (!owner) { + return; + } + + try { + const auto * format = CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sample)); + + if (!format || format->mFormatID != kAudioFormatLinearPCM || + format->mSampleRate != jni::SystemAudioCapture::kSampleRate || + format->mChannelsPerFrame != jni::SystemAudioCapture::kChannels || + !(format->mFormatFlags & kAudioFormatFlagIsFloat) || format->mBitsPerChannel != 32) { + owner->Fail("System audio: Unexpected ScreenCaptureKit PCM format"); + return; + } + + // Room for one buffer per channel of planar stereo. + struct { + AudioBufferList list; + AudioBuffer extra; + } storage {}; + CMBlockBufferRef block = nullptr; + + OSStatus status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sample, nullptr, + &storage.list, sizeof(storage), kCFAllocatorDefault, kCFAllocatorDefault, + kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &block); + + // Release the retained block even if allocating the conversion buffer fails. + std::unique_ptr retained(block, &CFRelease); + + if (status != noErr) { + owner->Fail("System audio: Cannot read ScreenCaptureKit PCM"); + return; + } + + const size_t frames = CMSampleBufferGetNumSamples(sample); + const bool planar = format->mFormatFlags & kAudioFormatFlagIsNonInterleaved; + const AudioBufferList & list = storage.list; + + if (list.mNumberBuffers != (planar ? 2U : 1U)) { + owner->Fail("System audio: Unexpected ScreenCaptureKit channel layout"); + return; + } + + for (UInt32 b = 0; b < list.mNumberBuffers; ++b) { + if (!list.mBuffers[b].mData || list.mBuffers[b].mDataByteSize < frames * sizeof(float) * (planar ? 1 : 2)) { + owner->Fail("System audio: Incomplete ScreenCaptureKit PCM"); + return; + } + } + + std::vector pcm(frames * 2); + + for (size_t f = 0; f < frames; ++f) { + for (size_t c = 0; c < 2; ++c) { + float value = static_cast(list.mBuffers[planar ? c : 0].mData)[planar ? f : f * 2 + c]; + + if (!std::isfinite(value)) { + value = 0; + } + + pcm[f * 2 + c] = static_cast(std::max(-32768.0f, std::min(32767.0f, value * 32768.0f))); + } + } + + // The presentation time is on the host clock; translate it to the + // clock of webrtc::TimeMicros(). + const double pts = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sample)); + const double now = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())); + + if (!std::isfinite(pts) || !std::isfinite(now)) { + owner->Fail("System audio: Invalid ScreenCaptureKit timestamp"); + return; + } + + owner->Append(pcm.data(), frames, webrtc::TimeMicros() + static_cast((pts - now) * webrtc::kNumMicrosecsPerSec)); + } + catch (...) { + owner->Fail("System audio: ScreenCaptureKit callback failed"); + } + } } + @end -namespace jni { -namespace { -struct Completion { - std::mutex mutex; - std::condition_variable condition; - bool done = false; - std::atomic abandoned{false}; - id keepAlive = nil; - NSError * error = nil; - SCShareableContent * content = nil; - ~Completion() { [keepAlive release]; [error release]; [content release]; } - void Finish(NSError * e, SCShareableContent * c = nil) noexcept { - try { - std::lock_guard lock(mutex); - error = [e retain]; content = [c retain]; done = true; - condition.notify_all(); - } - catch (...) { } // The waiting caller will time out; never unwind into the OS. - } - void Wait() { - std::unique_lock lock(mutex); - if (!condition.wait_for(lock, std::chrono::seconds(15), [&] { return done; })) { - abandoned = true; - throw Exception("System audio: ScreenCaptureKit timed out; check screen/audio recording permission"); - } - if (error) throw Exception([[error localizedDescription] UTF8String]); - } -}; -class API_AVAILABLE(macos(13.0)) ScreenAudioCapture final : public SystemAudioCapture { - SCStream * stream = nil; - SystemAudioReceiver * receiver = nil; - dispatch_queue_t queue = nullptr; -public: - ~ScreenAudioCapture() override { - @autoreleasepool { - if (receiver) { @synchronized(receiver) { receiver->owner = nullptr; } } - if (stream) { - auto completed = std::make_shared(); - completed->keepAlive = [receiver retain]; - [stream stopCaptureWithCompletionHandler:^(NSError * error) { completed->Finish(error); }]; - try { completed->Wait(); } catch (...) { } - [stream release]; - } - [receiver release]; - if (queue) dispatch_release(queue); - } - } - void Open() override { - @autoreleasepool { - auto content = std::make_shared(); - [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:YES - completionHandler:^(SCShareableContent * result, NSError * error) { content->Finish(error, result); }]; - content->Wait(); - if (!content->content || content->content.displays.count == 0) - throw Exception("System audio: no shareable display; grant screen/audio recording permission"); - SCContentFilter * filter = [[SCContentFilter alloc] initWithDisplay:content->content.displays.firstObject excludingWindows:@[]]; - SCStreamConfiguration * config = [[SCStreamConfiguration alloc] init]; - config.capturesAudio = YES; - config.excludesCurrentProcessAudio = YES; - config.sampleRate = 48000; - config.channelCount = 2; - config.width = 2; config.height = 2; - config.minimumFrameInterval = CMTimeMake(1, 1); - receiver = [[SystemAudioReceiver alloc] init]; - receiver->owner = this; - queue = dispatch_queue_create("webrtc-system-audio", DISPATCH_QUEUE_SERIAL); - stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:receiver]; - [filter release]; [config release]; - NSError * error = nil; - if (![stream addStreamOutput:receiver type:SCStreamOutputTypeAudio sampleHandlerQueue:queue error:&error]) - throw Exception(error ? [[error localizedDescription] UTF8String] : "System audio: cannot add ScreenCaptureKit audio output"); - auto completed = std::make_shared(); - completed->keepAlive = [receiver retain]; - SCStream * starting = stream; - [stream startCaptureWithCompletionHandler:^(NSError * error) { - // A permission prompt can outlive the caller's deadline. A late - // successful start must not leave a recording running invisibly. - if (completed->abandoned.load() && !error) { - // Keep the detached receiver alive until the late stream - // has stopped, even when the original stop timed out too. - [starting stopCaptureWithCompletionHandler:^(NSError * stopError) { - completed->Finish(stopError); - }]; - } - else completed->Finish(error); - }]; - completed->Wait(); - } - } - void Poll() override { } -}; -} -std::unique_ptr SystemAudioCapture::Create() { - if (@available(macOS 13.0, *)) return std::make_unique(); - throw Exception("System audio requires macOS 13 or later"); -} +namespace jni +{ + namespace + { + // How long a ScreenCaptureKit request may take, including a + // permission prompt. + constexpr std::chrono::seconds kRequestTimeout(15); + + // The completion of an asynchronous ScreenCaptureKit request. Its + // handler holds it, so it outlives a caller that gave up waiting. + struct Completion + { + std::mutex mutex; + std::condition_variable condition; + bool done = false; + std::atomic abandoned { false }; + id keepAlive = nil; + NSError * error = nil; + SCShareableContent * content = nil; + + ~Completion() + { + [keepAlive release]; + [error release]; + [content release]; + } + + void Finish(NSError * e, SCShareableContent * c = nil) noexcept + { + try { + std::lock_guard lock(mutex); + + error = [e retain]; + content = [c retain]; + done = true; + + condition.notify_all(); + } + catch (...) { + // The waiting caller will time out; never unwind into the OS. + } + } + + void Wait() + { + std::unique_lock lock(mutex); + + if (!condition.wait_for(lock, kRequestTimeout, [this]() { return done; })) { + abandoned = true; + + throw Exception("System audio: ScreenCaptureKit timed out; check screen/audio recording permission"); + } + if (error) { + throw Exception([[error localizedDescription] UTF8String]); + } + } + }; + + // Records the system audio through ScreenCaptureKit, without the + // audio of this process. + class API_AVAILABLE(macos(13.0)) ScreenCaptureKitCapture final : public SystemAudioCapture + { + public: + ~ScreenCaptureKitCapture() override; + + void Open() override; + void Poll() override; + + private: + SCStream * stream_ = nil; + SystemAudioReceiver * receiver_ = nil; + dispatch_queue_t queue_ = nullptr; + }; + + ScreenCaptureKitCapture::~ScreenCaptureKitCapture() + { + @autoreleasepool { + if (receiver_) { + @synchronized(receiver_) { + receiver_->owner = nullptr; + } + } + + if (stream_) { + auto completed = std::make_shared(); + completed->keepAlive = [receiver_ retain]; + + [stream_ stopCaptureWithCompletionHandler:^(NSError * error) { + completed->Finish(error); + }]; + + try { + completed->Wait(); + } + catch (...) { + } + + [stream_ release]; + } + + [receiver_ release]; + + if (queue_) { + dispatch_release(queue_); + } + } + } + + void ScreenCaptureKitCapture::Open() + { + @autoreleasepool { + auto content = std::make_shared(); + + [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:YES + completionHandler:^(SCShareableContent * result, NSError * error) { + content->Finish(error, result); + }]; + + content->Wait(); + + if (!content->content || content->content.displays.count == 0) { + throw Exception("System audio: No shareable display; grant screen/audio recording permission"); + } + + SCContentFilter * filter = [[SCContentFilter alloc] initWithDisplay:content->content.displays.firstObject + excludingWindows:@[]]; + SCStreamConfiguration * config = [[SCStreamConfiguration alloc] init]; + config.capturesAudio = YES; + config.excludesCurrentProcessAudio = YES; + config.sampleRate = kSampleRate; + config.channelCount = kChannels; + // The stream always captures video too; keep it as small and + // as rare as possible. + config.width = 2; + config.height = 2; + config.minimumFrameInterval = CMTimeMake(1, 1); + + receiver_ = [[SystemAudioReceiver alloc] init]; + receiver_->owner = this; + queue_ = dispatch_queue_create("webrtc-system-audio", DISPATCH_QUEUE_SERIAL); + stream_ = [[SCStream alloc] initWithFilter:filter configuration:config delegate:receiver_]; + + [filter release]; + [config release]; + + NSError * error = nil; + + if (![stream_ addStreamOutput:receiver_ type:SCStreamOutputTypeAudio sampleHandlerQueue:queue_ error:&error]) { + throw Exception(error + ? [[error localizedDescription] UTF8String] + : "System audio: Cannot add ScreenCaptureKit audio output"); + } + + auto completed = std::make_shared(); + completed->keepAlive = [receiver_ retain]; + SCStream * starting = stream_; + + [stream_ startCaptureWithCompletionHandler:^(NSError * startError) { + // A permission prompt can outlive the caller's deadline. A + // late successful start must not leave a recording running + // invisibly. Keep the detached receiver alive until the + // late stream has stopped, even when the original stop + // timed out too. + if (completed->abandoned.load() && !startError) { + [starting stopCaptureWithCompletionHandler:^(NSError * stopError) { + completed->Finish(stopError); + }]; + } + else { + completed->Finish(startError); + } + }]; + + completed->Wait(); + } + } + + void ScreenCaptureKitCapture::Poll() + { + // The receiver appends the audio as ScreenCaptureKit delivers it. + } + } + + std::unique_ptr SystemAudioCapture::Create() + { + if (@available(macOS 13.0, *)) { + return std::make_unique(); + } + + throw Exception("System audio requires macOS 13 or later"); + } } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp b/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp index 8ecb087ae..1ec5d4c0b 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/windows/SystemAudioCapture.cpp @@ -15,85 +15,165 @@ */ #include "media/audio/SystemAudioCapture.h" + #include "Exception.h" + #include "rtc_base/time_utils.h" + #include #include #include #include + #include -namespace jni { -namespace { -using Microsoft::WRL::ComPtr; -void Check(HRESULT hr, const char * operation) { - if (FAILED(hr)) { - std::ostringstream message; - message << "System audio: " << operation << " (HRESULT 0x" << std::hex - << static_cast(hr) << "); restart capture after an output change"; - throw Exception(message.str().c_str()); - } -} -class WasapiCapture final : public SystemAudioCapture { - bool com = false; - ComPtr enumerator; - ComPtr device; - ComPtr client; - ComPtr capture; -public: - ~WasapiCapture() override { - if (client) client->Stop(); - capture.Reset(); client.Reset(); device.Reset(); enumerator.Reset(); - if (com) CoUninitialize(); - } - void Open() override { - Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "initialize COM"); - com = true; - Check(CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, - IID_PPV_ARGS(enumerator.GetAddressOf())), "create output enumerator"); - Check(enumerator->GetDefaultAudioEndpoint(eRender, eMultimedia, device.GetAddressOf()), "get default output"); - Check(device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, - reinterpret_cast(client.GetAddressOf())), "activate output"); - WAVEFORMATEX format{}; - format.wFormatTag = WAVE_FORMAT_PCM; - format.nChannels = 2; - format.nSamplesPerSec = 48000; - format.wBitsPerSample = 16; - format.nBlockAlign = 4; - format.nAvgBytesPerSec = 192000; - Check(client->Initialize(AUDCLNT_SHAREMODE_SHARED, - AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | - AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, - 1000000, 0, &format, nullptr), "initialize loopback"); - Check(client->GetService(IID_PPV_ARGS(capture.GetAddressOf())), "get loopback capture"); - Check(client->Start(), "start loopback"); - } - void Poll() override { - for (int i = 0; i < 10; ++i) { - UINT32 frames = 0; - Check(capture->GetNextPacketSize(&frames), "query output packet"); - if (!frames) break; - BYTE * data = nullptr; - DWORD flags = 0; - UINT64 position = 0, qpc = 0; - Check(capture->GetBuffer(&data, &frames, &flags, &position, &qpc), "read output packet"); - try { - int64_t time = webrtc::TimeMicros() - frames * 1000000LL / 48000; - if (!(flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR)) { - LARGE_INTEGER now, frequency; - QueryPerformanceCounter(&now); - QueryPerformanceFrequency(&frequency); - const int64_t host_us = static_cast( - static_cast(now.QuadPart) * 1000000 / frequency.QuadPart); - time = webrtc::TimeMicros() + static_cast(qpc / 10) - host_us; - } - Append(flags & AUDCLNT_BUFFERFLAGS_SILENT ? nullptr : reinterpret_cast(data), frames, time); - } - catch (...) { capture->ReleaseBuffer(frames); throw; } - Check(capture->ReleaseBuffer(frames), "release output packet"); - } - } -}; -} -std::unique_ptr SystemAudioCapture::Create() { return std::make_unique(); } +namespace jni +{ + namespace + { + using Microsoft::WRL::ComPtr; + + // The length of the shared-mode loopback buffer, 100 ms in 100 ns units. + constexpr REFERENCE_TIME kBufferDuration = 1000000; + + // The most packets read by one Poll(), so that stopping stays + // responsive even while an output is very busy. + constexpr int kMaxPacketsPerPoll = 10; + + void ThrowIfFailed(HRESULT hr, const char * operation) + { + if (FAILED(hr)) { + std::ostringstream message; + message << "System audio: Cannot " << operation << " (HRESULT 0x" << std::hex + << static_cast(hr) << "); restart capture after an output change"; + + throw Exception(message.str().c_str()); + } + } + + // Records the default multimedia output through WASAPI loopback. The + // COM objects are created, used and released on the capture worker. + class WasapiLoopbackCapture final : public SystemAudioCapture + { + public: + ~WasapiLoopbackCapture() override; + + void Open() override; + void Poll() override; + + private: + // Converts a packet's performance counter position, in 100 ns + // units, to the clock of webrtc::TimeMicros(). + int64_t ToTimeMicros(UINT64 qpcPosition) const; + + bool comInitialized_ = false; + LARGE_INTEGER qpcFrequency_ {}; + ComPtr enumerator_; + ComPtr device_; + ComPtr client_; + ComPtr captureClient_; + }; + + WasapiLoopbackCapture::~WasapiLoopbackCapture() + { + if (client_) { + client_->Stop(); + } + + captureClient_.Reset(); + client_.Reset(); + device_.Reset(); + enumerator_.Reset(); + + if (comInitialized_) { + CoUninitialize(); + } + } + + void WasapiLoopbackCapture::Open() + { + ThrowIfFailed(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "initialize COM"); + comInitialized_ = true; + + QueryPerformanceFrequency(&qpcFrequency_); + + ThrowIfFailed(CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, + IID_PPV_ARGS(enumerator_.GetAddressOf())), "create the output enumerator"); + ThrowIfFailed(enumerator_->GetDefaultAudioEndpoint(eRender, eMultimedia, device_.GetAddressOf()), + "get the default output"); + ThrowIfFailed(device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, + reinterpret_cast(client_.GetAddressOf())), "activate the output"); + + WAVEFORMATEX format {}; + format.wFormatTag = WAVE_FORMAT_PCM; + format.nChannels = static_cast(kChannels); + format.nSamplesPerSec = kSampleRate; + format.wBitsPerSample = 16; + format.nBlockAlign = static_cast(format.nChannels * format.wBitsPerSample / 8); + format.nAvgBytesPerSec = format.nSamplesPerSec * format.nBlockAlign; + + // The audio engine converts the mix format of the output to this one. + ThrowIfFailed(client_->Initialize(AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | + AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, + kBufferDuration, 0, &format, nullptr), "initialize the loopback"); + ThrowIfFailed(client_->GetService(IID_PPV_ARGS(captureClient_.GetAddressOf())), + "get the loopback capture"); + ThrowIfFailed(client_->Start(), "start the loopback"); + } + + int64_t WasapiLoopbackCapture::ToTimeMicros(UINT64 qpcPosition) const + { + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + + const int64_t nowUs = static_cast( + static_cast(now.QuadPart) * webrtc::kNumMicrosecsPerSec / qpcFrequency_.QuadPart); + + return webrtc::TimeMicros() + static_cast(qpcPosition / 10) - nowUs; + } + + void WasapiLoopbackCapture::Poll() + { + for (int i = 0; i < kMaxPacketsPerPoll; ++i) { + UINT32 frames = 0; + + ThrowIfFailed(captureClient_->GetNextPacketSize(&frames), "query the output packet"); + + if (frames == 0) { + break; + } + + BYTE * data = nullptr; + DWORD flags = 0; + UINT64 devicePosition = 0; + UINT64 qpcPosition = 0; + + ThrowIfFailed(captureClient_->GetBuffer(&data, &frames, &flags, &devicePosition, &qpcPosition), + "read the output packet"); + + try { + int64_t captureUs = (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) + ? webrtc::TimeMicros() - frames * webrtc::kNumMicrosecsPerSec / kSampleRate + : ToTimeMicros(qpcPosition); + const int16_t * samples = (flags & AUDCLNT_BUFFERFLAGS_SILENT) + ? nullptr + : reinterpret_cast(data); + + Append(samples, frames, captureUs); + } + catch (...) { + captureClient_->ReleaseBuffer(frames); + throw; + } + + ThrowIfFailed(captureClient_->ReleaseBuffer(frames), "release the output packet"); + } + } + } + + std::unique_ptr SystemAudioCapture::Create() + { + return std::make_unique(); + } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java index 1817d739b..0ada01263 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java @@ -21,17 +21,17 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import java.util.concurrent.locks.LockSupport; +import java.util.function.Supplier; /** - * Captures the system output as a separate audio track source, at 48 kHz stereo. - * Use with {@code PeerConnectionFactory.createAudioTrack}, alongside a screen - * video track. A separate microphone track must also use pushed audio; + * Captures the system output as a separate audio track source, at 48 kHz + * stereo. Use it with {@code PeerConnectionFactory.createAudioTrack}, alongside + * a screen video track. A separate microphone track must also use pushed audio; * do not combine this with the factory's device-captured audio sources. *

- * Windows uses WASAPI loopback of the default multimedia output; Linux uses - * the default output's PulseAudio monitor (also available with pipewire-pulse). + * Windows uses WASAPI loopback of the default multimedia output; Linux uses the + * default output's PulseAudio monitor (also available with pipewire-pulse). * macOS 13 or later uses ScreenCaptureKit and excludes this process's audio. * Screen/audio recording permission is required on macOS. This class does not * capture a selected window's or process's audio. @@ -41,172 +41,326 @@ * recording and are available through {@link #getFailure()}. Remote call audio * played on the captured output is included on Windows and Linux: route it to * another output to prevent feedback. No microphone mixing or echo cancellation - * is performed. Do not call the inherited pushAudio methods during capture. + * is performed. The inherited {@code pushAudio} methods are rejected, since the + * captured audio is the only audio this source sends. + * + * @author Alex Andres */ public final class SystemAudioSource extends CustomAudioSource implements AutoCloseable { - private volatile boolean recording; - private volatile Throwable failure; - private volatile Thread worker; - private volatile boolean disposed; - private final AtomicBoolean released = new AtomicBoolean(); - private final Supplier backend; - - /** Creates an idle source. No device or recording permission is opened yet. */ - public SystemAudioSource() { - this(Capture::new); - } - - SystemAudioSource(Supplier backend) { - this.backend = backend; - } - - /** - * Starts capture, waiting for the platform to open the output. - * Calling this while already recording does nothing. Startup uses bounded platform waits, - * including the macOS permission prompt; use a background thread. - * - * @throws IllegalStateException if disposed or capture cannot start - */ - public void start() { - if (Thread.currentThread() == worker) { - throw new IllegalStateException("Start capture outside an audio callback"); - } - synchronized (this) { - startLocked(); - } - } - - private void startLocked() { - if (disposed) { - throw new IllegalStateException("System audio source is disposed"); - } - if (recording) { - return; - } - stopLocked(); - // The previous worker may have disposed the source from its last sink - // callback while stopLocked was joining it. - if (disposed) { - throw new IllegalStateException("System audio source is disposed"); - } - failure = null; - CompletableFuture started = new CompletableFuture<>(); - recording = true; - worker = new Thread(() -> capture(started), "webrtc-system-audio"); - worker.setDaemon(true); - try { - worker.start(); - started.join(); - } - catch (CompletionException e) { - stop(); - throw new IllegalStateException("Cannot start system audio capture", e.getCause()); - } - catch (RuntimeException | Error e) { - recording = false; - throw e; - } - } - - private void capture(CompletableFuture started) { - try (Backend capture = backend.get()) { - capture.initialize(); - started.complete(null); - while (recording) { - capture.pump(this); - LockSupport.parkNanos(2_000_000L); - } - } - catch (Throwable e) { - failure = e; - started.completeExceptionally(e); - } - finally { - recording = false; - if (disposed) releaseSource(); - } - } - - /** Stops and joins capture; the source and its tracks may be reused. */ - public void stop() { - // A local track sink can run inside pump(). It must not wait for its - // own worker or for a lifecycle caller that is joining that worker. - if (Thread.currentThread() == worker) { - recording = false; - return; - } - synchronized (this) { - stopLocked(); - } - } - - private void stopLocked() { - recording = false; - boolean interrupted = false; - if (worker != null) { - LockSupport.unpark(worker); - while (worker.isAlive()) { - try { - worker.join(); - } - catch (InterruptedException e) { - interrupted = true; - } - } - worker = null; - } - if (interrupted) { - Thread.currentThread().interrupt(); - } - } - - /** @return whether capture is running (including startup) */ - public boolean isRecording() { - return recording; - } - - /** @return the last startup/runtime failure, or null; cleared by start */ - public Throwable getFailure() { - return failure; - } - - /** Stops capture before releasing the native source. Safe to call repeatedly. */ - @Override - public void dispose() { - if (Thread.currentThread() == worker) { - disposed = true; - recording = false; - return; - } - synchronized (this) { - disposed = true; - stopLocked(); - releaseSource(); - } - } - - /** Stops capture and releases the source. */ - @Override - public void close() { - dispose(); - } - - private void releaseSource() { - if (released.compareAndSet(false, true)) super.dispose(); - } - - interface Backend extends AutoCloseable { - void initialize(); - void pump(CustomAudioSource source); - void dispose(); - default void close() { dispose(); } - } - - // Open, poll and close on the same thread (required by WASAPI's COM objects). - private static final class Capture extends DisposableNativeObject implements Backend { - public native void initialize(); - public native void pump(CustomAudioSource source); - public native void dispose(); - } + /** + * How long the capture worker waits between two polls of the platform. A + * chunk lasts 10 ms, and the platform buffers up to 100 ms. + */ + private static final long POLL_INTERVAL_NANOS = 2_000_000L; + + /** Creates the platform capture, once for each start. */ + private final Supplier backend; + + /** Set once the native source has been released. */ + private final AtomicBoolean released = new AtomicBoolean(); + + /** Whether capture is running, including its startup. */ + private volatile boolean recording; + + /** The last startup or runtime failure, cleared by a start. */ + private volatile Throwable failure; + + /** The thread that runs the capture, or null before the first start. */ + private volatile Thread worker; + + /** Set by the first disposal; never reset. */ + private volatile boolean disposed; + + + /** + * Creates an idle source. No device is opened and no recording permission + * is requested until {@link #start()}. + */ + public SystemAudioSource() { + this(Capture::new); + } + + /** + * Creates an idle source with the given platform capture, which lets tests + * replace the native one. + * + * @param backend Creates the platform capture, once for each start. + */ + SystemAudioSource(Supplier backend) { + this.backend = backend; + } + + /** + * Starts capture, waiting for the platform to open the output. Calling this + * while already recording does nothing. Startup uses bounded platform + * waits, including the macOS permission prompt, so call it on a background + * thread. + * + * @throws IllegalStateException If this source is disposed, if called from + * an audio callback of this source, or if + * capture cannot start; the platform error is + * then the cause. + */ + public void start() { + if (Thread.currentThread() == worker) { + throw new IllegalStateException("Start capture outside an audio callback"); + } + + synchronized (this) { + startLocked(); + } + } + + /** + * Stops and joins capture. The source and its tracks may be started again. + * Called from an audio callback of this source, it requests the stop + * without waiting for it. + */ + public void stop() { + // A local track sink can run inside pump(). It must not wait for its + // own worker or for a lifecycle caller that is joining that worker. + if (Thread.currentThread() == worker) { + recording = false; + return; + } + + synchronized (this) { + stopLocked(); + } + } + + /** + * Returns whether capture is running, including its startup. + * + * @return True while capturing. + */ + public boolean isRecording() { + return recording; + } + + /** + * Returns the last startup or runtime failure. Starting again clears it. + * + * @return The failure, or {@code null} if capture has not failed. + */ + public Throwable getFailure() { + return failure; + } + + /** + * Rejected: this source sends the captured system audio only. + * + * @throws IllegalStateException Always, unless called by the capture + * itself. + */ + @Override + public void pushAudio(byte[] audioData, int bits_per_sample, int sampleRate, + int channels, int frameCount) { + requireCaptureThread(); + + super.pushAudio(audioData, bits_per_sample, sampleRate, channels, frameCount); + } + + /** + * Rejected: this source sends the captured system audio only. + * + * @throws IllegalStateException Always, unless called by the capture + * itself. + */ + @Override + public void pushAudio(byte[] audioData, int bits_per_sample, int sampleRate, + int channels, int frameCount, long timestampUs) { + requireCaptureThread(); + + super.pushAudio(audioData, bits_per_sample, sampleRate, channels, frameCount, timestampUs); + } + + /** + * Stops capture before releasing the native source. Safe to call more than + * once. Called from an audio callback of this source, it requests the stop + * and the source is released once the callback has returned. + */ + @Override + public void dispose() { + if (Thread.currentThread() == worker) { + disposed = true; + recording = false; + return; + } + + synchronized (this) { + disposed = true; + + stopLocked(); + releaseSource(); + } + } + + /** + * Stops capture and releases the source, see {@link #dispose()}. + */ + @Override + public void close() { + dispose(); + } + + private void startLocked() { + if (disposed) { + throw new IllegalStateException("System audio source is disposed"); + } + if (recording) { + return; + } + + stopLocked(); + + // The previous worker may have disposed the source from its last sink + // callback while stopLocked() was joining it. + if (disposed) { + throw new IllegalStateException("System audio source is disposed"); + } + + CompletableFuture started = new CompletableFuture<>(); + + failure = null; + recording = true; + worker = new Thread(() -> capture(started), "webrtc-system-audio"); + worker.setDaemon(true); + + try { + worker.start(); + started.join(); + } + catch (CompletionException e) { + stopLocked(); + + throw new IllegalStateException("Cannot start system audio capture", e.getCause()); + } + catch (RuntimeException | Error e) { + recording = false; + + throw e; + } + } + + private void stopLocked() { + recording = false; + + Thread thread = worker; + boolean interrupted = false; + + if (thread != null) { + LockSupport.unpark(thread); + + while (thread.isAlive()) { + try { + thread.join(); + } + catch (InterruptedException e) { + interrupted = true; + } + } + + worker = null; + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** + * Runs on the worker: opens, polls and closes one platform capture, all on + * this thread, as WASAPI's COM objects require. + * + * @param started Completed once the capture is open, or with its failure. + */ + private void capture(CompletableFuture started) { + try (Backend capture = backend.get()) { + capture.initialize(); + + started.complete(null); + + while (recording) { + capture.pump(this); + + LockSupport.parkNanos(POLL_INTERVAL_NANOS); + } + } + catch (Throwable e) { + failure = e; + + started.completeExceptionally(e); + } + finally { + recording = false; + + if (disposed) { + releaseSource(); + } + } + } + + private void requireCaptureThread() { + if (Thread.currentThread() != worker) { + throw new IllegalStateException("A SystemAudioSource sends the captured system audio only. " + + "Push audio into a separate CustomAudioSource."); + } + } + + private void releaseSource() { + if (released.compareAndSet(false, true)) { + super.dispose(); + } + } + + + /** + * The platform capture of one start, opened, pumped and closed on the + * capture worker. + */ + interface Backend extends AutoCloseable { + + /** + * Opens the default output. + */ + void initialize(); + + /** + * Pushes the audio captured since the last call into the source. + * + * @param source The source to push into. + */ + void pump(CustomAudioSource source); + + /** + * Closes the output. + */ + void dispose(); + + @Override + default void close() { + dispose(); + } + + } + + + /** + * The native platform capture. + */ + private static final class Capture extends DisposableNativeObject implements Backend { + + @Override + public native void initialize(); + + @Override + public native void pump(CustomAudioSource source); + + @Override + public native void dispose(); + + } + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java index 3eec146a2..746e51cde 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceIntegrationTest.java @@ -16,43 +16,64 @@ package dev.onvoid.webrtc.media.audio; +import static org.junit.jupiter.api.Assertions.*; + import dev.onvoid.webrtc.TestBase; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.jupiter.api.Assertions.*; -/** Opt in with -Dtest.systemAudio=true while playing audio on the default output. */ +/** + * Captures the real system output. Opt in with {@code -Dtest.systemAudio=true} + * while audio plays on the default output. + */ @EnabledIfSystemProperty(named = "test.systemAudio", matches = "true") class SystemAudioSourceIntegrationTest extends TestBase { + @Test void capturesOutputAndRestarts() throws Exception { try (SystemAudioSource source = new SystemAudioSource()) { AudioTrack track = factory.createAudioTrack("system", source); AtomicReference received = new AtomicReference<>(new CountDownLatch(5)); + try { track.addSink((data, bits, rate, channels, frames) -> { - if (bits == 16 && rate == 48000 && channels == 2 && frames == 480) { - for (byte sample : data) { - if (sample != 0) { received.get().countDown(); break; } + if (bits != 16 || rate != 48000 || channels != 2 || frames != 480) { + return; + } + + for (byte sample : data) { + if (sample != 0) { + received.get().countDown(); + break; } } }); + source.start(); assertTrue(received.get().await(10, TimeUnit.SECONDS), "Play audio on the default output"); + source.stop(); assertNull(source.getFailure()); assertFalse(source.isRecording()); + received.set(new CountDownLatch(5)); + source.start(); assertTrue(received.get().await(10, TimeUnit.SECONDS), "No audio after restart"); assertTrue(source.isRecording()); + source.stop(); assertNull(source.getFailure()); } - finally { track.dispose(); } + finally { + track.dispose(); + } } } + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java index 5e52424fe..f4b1521ba 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/SystemAudioSourceTest.java @@ -16,30 +16,27 @@ package dev.onvoid.webrtc.media.audio; +import static org.junit.jupiter.api.Assertions.*; + import dev.onvoid.webrtc.TestBase; + import org.junit.jupiter.api.Test; -import java.util.concurrent.CountDownLatch; + import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.jupiter.api.Assertions.*; +/** + * Lifecycle tests of {@link SystemAudioSource}, with backends that stand in for + * the platform capture, so that no audio hardware is needed. + */ class SystemAudioSourceTest extends TestBase { - private static class Backend implements SystemAudioSource.Backend { - final CountDownLatch pumped = new CountDownLatch(1); - final CountDownLatch closed = new CountDownLatch(1); - Thread thread; - public void initialize() { thread = Thread.currentThread(); } - public void pump(CustomAudioSource source) { - assertSame(thread, Thread.currentThread()); - pumped.countDown(); - } - public void dispose() { - assertSame(thread, Thread.currentThread()); - closed.countDown(); - } - } + + /** One 10 ms chunk of 48 kHz stereo silence. */ + private static final byte[] CHUNK = new byte[480 * 2 * 2]; + @Test void restartAndDispose() throws Exception { @@ -47,37 +44,53 @@ void restartAndDispose() throws Exception { AtomicReference latest = new AtomicReference<>(); SystemAudioSource source = new SystemAudioSource(() -> { opened.incrementAndGet(); + Backend backend = new Backend(); latest.set(backend); + return backend; }); + try { assertFalse(source.isRecording()); + source.stop(); source.start(); + assertTrue(latest.get().pumped.await(5, TimeUnit.SECONDS)); + + // Starting a running source does not open the output again. source.start(); assertEquals(1, opened.get()); + source.stop(); assertEquals(0, latest.get().closed.getCount()); assertFalse(source.isRecording()); + source.start(); assertEquals(2, opened.get()); assertNull(source.getFailure()); } - finally { source.dispose(); } + finally { + source.dispose(); + } + source.dispose(); + assertThrows(IllegalStateException.class, source::start); - assertThrows(IllegalStateException.class, - () -> source.pushAudio(new byte[1920], 16, 48000, 2, 480)); } @Test void startupFailureClosesBackend() { IllegalStateException denied = new IllegalStateException("permission denied"); Backend backend = new Backend() { - public void initialize() { super.initialize(); throw denied; } + @Override + public void initialize() { + super.initialize(); + throw denied; + } }; + try (SystemAudioSource source = new SystemAudioSource(() -> backend)) { assertSame(denied, assertThrows(IllegalStateException.class, source::start).getCause()); assertSame(denied, source.getFailure()); @@ -90,25 +103,54 @@ void startupFailureClosesBackend() { void runtimeFailureCanRestart() throws Exception { IllegalStateException lost = new IllegalStateException("output lost"); Backend broken = new Backend() { - public void pump(CustomAudioSource source) { throw lost; } + @Override + public void pump(CustomAudioSource source) { + throw lost; + } }; AtomicInteger attempts = new AtomicInteger(); + try (SystemAudioSource source = new SystemAudioSource( () -> attempts.getAndIncrement() == 0 ? broken : new Backend())) { source.start(); + assertTrue(broken.closed.await(5, TimeUnit.SECONDS)); - source.stop(); // Join also waits for publication of the failure. + + // The join also waits for the failure to be published. + source.stop(); assertSame(lost, source.getFailure()); + source.start(); assertNull(source.getFailure()); assertTrue(source.isRecording()); } } + @Test + void pushFromOtherThreadRejected() throws Exception { + Backend backend = new Backend(); + + try (SystemAudioSource source = new SystemAudioSource(() -> backend)) { + // Pushed audio would interleave with the captured audio. + assertThrows(IllegalStateException.class, + () -> source.pushAudio(CHUNK, 16, 48000, 2, 480)); + + source.start(); + assertTrue(backend.pumped.await(5, TimeUnit.SECONDS)); + + assertThrows(IllegalStateException.class, + () -> source.pushAudio(CHUNK, 16, 48000, 2, 480)); + assertThrows(IllegalStateException.class, + () -> source.pushAudio(CHUNK, 16, 48000, 2, 480, 1)); + assertTrue(source.isRecording()); + } + } + @Test void callbackCanDisposeSource() throws Exception { AtomicReference ref = new AtomicReference<>(); Backend backend = new Backend() { + @Override public void pump(CustomAudioSource source) { ref.get().dispose(); super.pump(source); @@ -116,13 +158,18 @@ public void pump(CustomAudioSource source) { }; SystemAudioSource source = new SystemAudioSource(() -> backend); ref.set(source); + try { source.start(); + assertTrue(backend.closed.await(5, TimeUnit.SECONDS)); + source.stop(); assertThrows(IllegalStateException.class, source::start); } - finally { source.dispose(); } + finally { + source.dispose(); + } } @Test @@ -130,36 +177,56 @@ void disposeDuringNativeCallback() throws Exception { CountDownLatch entered = new CountDownLatch(1); CountDownLatch disposing = new CountDownLatch(1); Backend backend = new Backend() { + @Override public void pump(CustomAudioSource source) { - source.pushAudio(new byte[1920], 16, 48000, 2, 480); + source.pushAudio(CHUNK, 16, 48000, 2, 480); } }; SystemAudioSource source = new SystemAudioSource(() -> backend); AudioTrack track = factory.createAudioTrack("system-callback", source); CompletableFuture disposed = new CompletableFuture<>(); + try { track.addSink((data, bits, rate, channels, frames) -> { entered.countDown(); + try { - if (!disposing.await(5, TimeUnit.SECONDS)) return; + if (!disposing.await(5, TimeUnit.SECONDS)) { + return; + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; } - catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } + source.dispose(); }); + source.start(); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + Thread closer = new Thread(() -> { // Hold the lifecycle monitor before allowing the native sink // callback to request disposal on the capture worker as well. synchronized (source) { disposing.countDown(); - try { source.dispose(); disposed.complete(null); } - catch (Throwable e) { disposed.completeExceptionally(e); } + + try { + source.dispose(); + disposed.complete(null); + } + catch (Throwable e) { + disposed.completeExceptionally(e); + } } }); closer.setDaemon(true); closer.start(); + disposed.get(5, TimeUnit.SECONDS); + assertFalse(source.isRecording()); assertNull(source.getFailure()); } @@ -174,22 +241,65 @@ public void pump(CustomAudioSource source) { void sourceFeedsNativeTrack() throws Exception { CountDownLatch received = new CountDownLatch(1); Backend backend = new Backend() { + @Override public void pump(CustomAudioSource source) { - source.pushAudio(new byte[1920], 16, 48000, 2, 480); + source.pushAudio(CHUNK, 16, 48000, 2, 480); } }; + try (SystemAudioSource source = new SystemAudioSource(() -> backend)) { AudioTrack track = factory.createAudioTrack("system", source); + try { track.addSink((data, bits, rate, channels, frames) -> { - if (bits == 16 && rate == 48000 && channels == 2 && frames == 480) + if (bits == 16 && rate == 48000 && channels == 2 && frames == 480) { received.countDown(); + } }); + source.start(); + assertTrue(received.await(5, TimeUnit.SECONDS)); + source.stop(); } - finally { track.dispose(); } + finally { + track.dispose(); + } } } + + + /** + * Stands in for the platform capture and checks that it is only used on + * the thread that opened it. + */ + private static class Backend implements SystemAudioSource.Backend { + + final CountDownLatch pumped = new CountDownLatch(1); + + final CountDownLatch closed = new CountDownLatch(1); + + Thread thread; + + + @Override + public void initialize() { + thread = Thread.currentThread(); + } + + @Override + public void pump(CustomAudioSource source) { + assertSame(thread, Thread.currentThread()); + pumped.countDown(); + } + + @Override + public void dispose() { + assertSame(thread, Thread.currentThread()); + closed.countDown(); + } + + } + } From 986a9bff592020aed802e9743712415dd9c69638 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 8 Nov 2026 15:08:26 +0100 Subject: [PATCH 3/3] docs: send system audio beside a microphone track Now that a factory sends device-captured and pushed audio side by side, the guide and javadoc send the microphone from the same factory instead of a workaround, and a test sends both on one connection. --- docs/guide/audio/system-audio-source.md | 25 +++-- .../webrtc/media/audio/SystemAudioSource.java | 4 +- .../onvoid/webrtc/DeviceAudioSendTests.java | 102 ++++++++++++------ .../media/audio/TestSystemAudioSources.java | 72 +++++++++++++ 4 files changed, 157 insertions(+), 46 deletions(-) create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/TestSystemAudioSources.java diff --git a/docs/guide/audio/system-audio-source.md b/docs/guide/audio/system-audio-source.md index fac7ab893..295f710ab 100644 --- a/docs/guide/audio/system-audio-source.md +++ b/docs/guide/audio/system-audio-source.md @@ -85,17 +85,20 @@ screen capture, output-device, and receiver latency. ## Microphone and feedback -A factory that sends pushed audio cannot also send its own device-captured audio -from `factory.createAudioSource(...)`. The existing factory guard applies to -`SystemAudioSource` too. For a microphone alongside system audio on the same -connection, capture the microphone independently (for example with -`AudioRecorder`) and feed a **separate** `CustomAudioSource`, so both tracks use -pushed audio. Respect that source's 16-bit PCM and 10 ms chunk requirements. Stop -the recorder before disposing its custom source. Alternatively, use separate -factories and connections for device-captured microphone and system audio. - -No microphone mixing or echo cancellation is added here. Keep microphone and -system audio on separate tracks, and use headphones to prevent acoustic feedback. +Send the microphone as a second track of the same factory and connection. Each +sender gets the audio of its own track, so neither track hears the other: + +```java +AudioTrackSource microphone = factory.createAudioSource(new AudioOptions()); +AudioTrack microphoneTrack = factory.createAudioTrack("microphone", microphone); +peerConnection.addTrack(microphoneTrack, Collections.singletonList("presentation")); +``` + +The microphone track gets the factory's echo cancellation, noise suppression and +gain control as its `AudioOptions` ask for; the system audio track gets none, and +the two are not mixed. Echo cancellation removes what this application plays, not +what other applications play, so system audio that reaches the microphone through +the speakers is sent twice: use headphones to prevent that. On Windows and Linux, route received call audio to another output or to a virtual output that is not monitored. Headphones alone do not prevent a loopback source from capturing remote audio played on the monitored output. On macOS this process's diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java index 0ada01263..4fe38473c 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/SystemAudioSource.java @@ -27,8 +27,8 @@ /** * Captures the system output as a separate audio track source, at 48 kHz * stereo. Use it with {@code PeerConnectionFactory.createAudioTrack}, alongside - * a screen video track. A separate microphone track must also use pushed audio; - * do not combine this with the factory's device-captured audio sources. + * a screen video track, and beside a microphone track of the same factory, if + * any: each sender gets the audio of its own track. *

* Windows uses WASAPI loopback of the default multimedia output; Linux uses the * default output's PulseAudio monitor (also available with pipewire-pulse). diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java index 3efbee0c7..65d2b20de 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/DeviceAudioSendTests.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import dev.onvoid.webrtc.media.audio.AudioOptions; @@ -25,6 +26,8 @@ import dev.onvoid.webrtc.media.audio.AudioTrackSink; import dev.onvoid.webrtc.media.audio.AudioTrackSource; import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.audio.SystemAudioSource; +import dev.onvoid.webrtc.media.audio.TestSystemAudioSources; import dev.onvoid.webrtc.media.audio.ToneAudioDeviceModule; import org.junit.jupiter.api.Test; @@ -146,45 +149,78 @@ void deviceAndPushedTracksOnOnePeer() throws Exception { // The callee's transceivers follow the order of the offer: the // microphone first, then the pushed silence. - RTCRtpReceiver[] receivers = call.getReceivers(); - AudioTrack microphone = (AudioTrack) receivers[0].getTrack(); - AudioTrack pushed = (AudioTrack) receivers[1].getTrack(); - - CountDownLatch microphoneAudible = new CountDownLatch(5); - CountDownLatch pushedReceived = new CountDownLatch(50); - AtomicBoolean pushedAudible = new AtomicBoolean(); - - AudioTrackSink microphoneSink = (data, bits, rate, channels, frames) -> { - if (isAudible(data, bits, channels, frames)) { - microphoneAudible.countDown(); - } - }; - // Recorded audio fed into the pushed track's sender as well would - // make the silence audible. - AudioTrackSink pushedSink = (data, bits, rate, channels, frames) -> { - if (isAudible(data, bits, channels, frames)) { - pushedAudible.set(true); - } - pushedReceived.countDown(); - }; + assertOnlyFirstAudible(call.getReceivers()); + } + } - microphone.addSink(microphoneSink); - pushed.addSink(pushedSink); + @Test + void microphoneAndSystemAudioOnOnePeer() throws Exception { + try (DeviceCall call = new DeviceCall(new AudioOptions()); + SystemAudioSource systemAudio = TestSystemAudioSources.silent()) { + AudioTrack systemTrack = call.factory.createAudioTrack("system", systemAudio); + RTCRtpSender systemSender = call.caller.getPeerConnection().addTrack(systemTrack, + Collections.singletonList("stream")); try { - assertTrue(microphoneAudible.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), - "No recorded audio at the callee"); - assertTrue(pushedReceived.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), - "No pushed audio at the callee"); - assertFalse(pushedAudible.get(), "Recorded audio reached the pushed track"); + call.connect(); + systemAudio.start(); + + // The microphone first, then the silent system output. + assertOnlyFirstAudible(call.getReceivers()); + assertNull(systemAudio.getFailure()); } finally { - microphone.removeSink(microphoneSink); - pushed.removeSink(pushedSink); + systemAudio.stop(); + systemSender.dispose(); - for (RTCRtpReceiver receiver : receivers) { - receiver.dispose(); - } + // The connection holds the track until it is closed. + call.caller.close(); + systemTrack.dispose(); + } + } + } + + /** + * Checks that the first of two received tracks carries the recorded tone + * and the second only silence, which recorded audio fed into the second + * track's sender as well would make audible. Disposes the receivers. + */ + private static void assertOnlyFirstAudible(RTCRtpReceiver[] receivers) throws InterruptedException { + AudioTrack microphone = (AudioTrack) receivers[0].getTrack(); + AudioTrack silent = (AudioTrack) receivers[1].getTrack(); + + CountDownLatch microphoneAudible = new CountDownLatch(5); + CountDownLatch silentReceived = new CountDownLatch(50); + AtomicBoolean silentAudible = new AtomicBoolean(); + + AudioTrackSink microphoneSink = (data, bits, rate, channels, frames) -> { + if (isAudible(data, bits, channels, frames)) { + microphoneAudible.countDown(); + } + }; + AudioTrackSink silentSink = (data, bits, rate, channels, frames) -> { + if (isAudible(data, bits, channels, frames)) { + silentAudible.set(true); + } + silentReceived.countDown(); + }; + + microphone.addSink(microphoneSink); + silent.addSink(silentSink); + + try { + assertTrue(microphoneAudible.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), + "No recorded audio at the callee"); + assertTrue(silentReceived.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), + "No audio of the second track at the callee"); + assertFalse(silentAudible.get(), "Recorded audio reached the second track"); + } + finally { + microphone.removeSink(microphoneSink); + silent.removeSink(silentSink); + + for (RTCRtpReceiver receiver : receivers) { + receiver.dispose(); } } } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/TestSystemAudioSources.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/TestSystemAudioSources.java new file mode 100644 index 000000000..4dd6aa102 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/TestSystemAudioSources.java @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +/** + * Creates {@link SystemAudioSource}s whose platform capture is replaced, for + * tests outside this package that need system audio without audio hardware. + * + * @author Alex Andres + */ +public final class TestSystemAudioSources { + + private TestSystemAudioSources() { + // Static access only. + } + + /** + * Creates a source whose capture delivers silence in real time, 10 ms of + * 48 kHz stereo every 10 ms, as a system output that plays nothing does. + * + * @return The source, not started. + */ + public static SystemAudioSource silent() { + return new SystemAudioSource(SilentCapture::new); + } + + + + private static final class SilentCapture implements SystemAudioSource.Backend { + + private static final long CHUNK_NANOS = 10_000_000L; + + private final byte[] chunk = new byte[480 * 2 * 2]; + + private long next; + + + @Override + public void initialize() { + next = System.nanoTime(); + } + + @Override + public void pump(CustomAudioSource source) { + while (System.nanoTime() >= next) { + source.pushAudio(chunk, 16, 48000, 2, 480); + + next += CHUNK_NANOS; + } + } + + @Override + public void dispose() { + } + + } + +}