From 0e6f1b28297f77346fbe7551c0fdf564d5ef717a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 12 Nov 2026 17:57:15 +0100 Subject: [PATCH 1/3] feat: let the user choose a screen or window in the macOS system picker VideoDesktopSource.setUseSystemPicker(true) makes start() show the operating system's own picker, instead of capturing the source set with setSourceId(). On macOS 14 and newer that is ScreenCaptureKit's SCContentSharingPicker, offering a screen or one or more windows; under Wayland it is the desktop portal, as before. Both come through DesktopCapturer::CreateGenericCapturer. ScreenCaptureKit draws the cursor, which DesktopAndCursorComposer would misplace on a picked window. Where isSystemPickerAvailable() is false, the source ends instead of capturing something the user did not choose. The default is unchanged. A source now ends when its capturer gives up for good: the user cancelled the picker or stopped sharing, or the captured window was closed. Before, capture stopped but the track stayed live. The ended listeners run on the capture thread, as for a missing backend, so that thread now keeps the source alive until it ends, and stop() called on it does not join it (#186). Should the last reference go there, a short-lived thread joins it. dispose() ends the capture without waiting, even while a track still holds the source. DesktopPickerExample shows the picker, and the desktop capture guide describes it. --- docs/guide/examples.md | 9 + docs/guide/video/desktop-capture.md | 33 +++- .../webrtc/examples/DesktopPickerExample.java | 94 ++++++++++ .../main/cpp/include/JNI_VideoDesktopSource.h | 16 ++ .../media/video/VideoTrackDesktopSource.h | 13 ++ .../main/cpp/src/JNI_VideoDesktopSource.cpp | 21 +++ .../media/video/VideoTrackDesktopSource.cpp | 114 ++++++++++++- .../media/video/VideoDesktopSource.java | 42 +++++ .../media/video/VideoDesktopSourceTests.java | 161 +++++++++++++++++- 9 files changed, 486 insertions(+), 17 deletions(-) create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java diff --git a/docs/guide/examples.md b/docs/guide/examples.md index c304b5f0a..7828bf2ed 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -148,6 +148,15 @@ Each audio sender gets the audio of its own track, so the microphone keeps its e mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ScreenShareAudioExample" ``` +## Desktop Picker + +The [`DesktopPickerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java) lets the user choose a screen or window in the operating system's own picker, on macOS 14 and newer and under Wayland. + +**Key features demonstrated:** +- Checking whether the system offers a picker +- Showing the picker by starting a `VideoDesktopSource` +- Reacting to the end of the track when the picker is cancelled or sharing stops + ## Media File The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index 9712ed7be..f3d1f26ad 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -70,6 +70,27 @@ videoSource.setSourceId(windowId, true); videoSource.start(); ``` +### Letting the User Choose in the System Picker + +Instead of listing sources yourself, you can let the user choose a screen or window in the operating system's own picker. It is available on macOS 14 and newer, where it is ScreenCaptureKit's content sharing picker, and on Linux under Wayland, where the desktop portal's picker is always used: + +```java +VideoDesktopSource videoSource = new VideoDesktopSource(); +videoSource.setFrameRate(30); + +if (VideoDesktopSource.isSystemPickerAvailable()) { + // The picker chooses the source; no setSourceId() is needed. + videoSource.setUseSystemPicker(true); +} +else { + videoSource.setSourceId(screenId, false); +} +``` + +`start()` shows the picker and returns at once. No frames arrive until the user has made a choice. The user can change it later from the macOS menu bar, and the frames follow. If the user cancels the picker, or stops sharing from the menu bar, the source ends (see [Handling the End of Capture](#handling-the-end-of-capture)). Where `isSystemPickerAvailable()` is false, a source set to use the picker ends right away instead of capturing. + +On macOS the picker offers a whole screen or one or more windows. Show it for one source at a time: sources waiting on it at the same time all receive the user's choice. The [`DesktopPickerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java) shows the picker and reports the frames it receives. + ### Creating a Video Track with the Desktop Source Once you have configured your desktop video source, you can create a video track: @@ -153,16 +174,20 @@ videoSource.stop(); videoSource.dispose(); ``` -### Handling Capture Failures +### Handling the End of Capture -`start()` returns before capturing begins. If capturing cannot begin, because no capture backend is available or the selected source does not exist, the source ends and its tracks with it. Create the track and add the listener before calling `start()`, as a listener added later misses this: +`start()` returns before capturing begins. The source ends, and its tracks with it, when capturing cannot begin or cannot go on: no capture backend is available, the selected source does not exist, the captured window was closed, or the user cancelled the system picker or stopped sharing. Create the track and add the listener before calling `start()`, as a listener added later misses an end that comes at once: ```java videoTrack.addTrackEndedListener(track -> { - // No frames will arrive from this source. + // No more frames will arrive from this source. The listener runs on the + // capture thread; stopping or disposing the source here is safe. + videoSource.stop(); }); ``` +`dispose()` ends the capture as well, even while a track still holds the source. + ### Processing Frames A `VideoDesktopSource` can pass its frames through a `VideoProcessor` before they reach its tracks, for example to blank out a part of the screen. It works as for a camera; see [Changing Frames Before They Are Sent](/guide/video/camera-capture#changing-frames-before-they-are-sent). @@ -188,7 +213,7 @@ windowCapturer.dispose(); On macOS 14 and newer, screens and windows are captured with ScreenCaptureKit; older releases use the earlier Core Graphics capture. Both need the app to have Screen Recording permission (System Settings → Privacy & Security → Screen & System Audio Recording). Without it, the windows of other apps are listed without their titles and their content cannot be captured. Windows of the app's own process need no permission, and are captured with Core Graphics. -While a captured window is minimized or hidden, its last image repeats. When it is closed, capture stops. +While a captured window is minimized or hidden, its last image repeats. When it is closed, the source ends. ## Conclusion diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java new file mode 100644 index 000000000..a2aa59a90 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.video.VideoDesktopSource; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +/** + * Example letting the user choose a screen or window in the operating + * system's own picker, instead of listing sources in the application. + *

+ * On macOS 14 and newer, {@link VideoDesktopSource#start()} shows + * ScreenCaptureKit's content sharing picker. The example prints the number and + * size of the frames it receives, and reports when the track ends: when the + * picker is cancelled, or the user stops sharing from the menu bar. + *

+ * Run it with: + *

+ * mvn -pl webrtc-examples exec:java -Dexec.mainClass=dev.onvoid.webrtc.examples.DesktopPickerExample
+ * 
+ * + * @author Alex Andres + */ +public class DesktopPickerExample { + + public static void main(String[] args) throws Exception { + if (!VideoDesktopSource.isSystemPickerAvailable()) { + System.out.println("This system offers no picker for screens and windows."); + return; + } + + PeerConnectionFactory factory = new PeerConnectionFactory(); + VideoDesktopSource videoSource = new VideoDesktopSource(); + videoSource.setFrameRate(15); + videoSource.setUseSystemPicker(true); + + VideoTrack videoTrack = factory.createVideoTrack("video0", videoSource); + + AtomicInteger frames = new AtomicInteger(); + AtomicReference size = new AtomicReference<>("none"); + CountDownLatch ended = new CountDownLatch(1); + + VideoTrackSink sink = frame -> { + size.set(frame.buffer.getWidth() + "x" + frame.buffer.getHeight()); + frames.incrementAndGet(); + frame.release(); + }; + + // Add the listener before start(), so that it hears of a picker that + // is cancelled at once. + videoTrack.addTrackEndedListener(track -> { + System.out.println("The track ended: the picker was cancelled, or sharing stopped."); + ended.countDown(); + }); + videoTrack.addSink(sink); + + System.out.println("Choose a screen or window in the picker. Press Ctrl+C to quit."); + videoSource.start(); + + try { + while (!ended.await(1, TimeUnit.SECONDS)) { + System.out.printf("Frames: %d, last size: %s%n", frames.get(), size.get()); + } + } + finally { + videoSource.stop(); + videoTrack.removeSink(sink); + videoTrack.dispose(); + videoSource.dispose(); + factory.dispose(); + } + } +} diff --git a/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h b/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h index dee525209..dc61874e4 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h @@ -31,6 +31,22 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_setMaxFrameSize (JNIEnv*, jobject, jint, jint); + /* + * Class: dev_onvoid_webrtc_media_video_VideoDesktopSource + * Method: setUseSystemPicker + * Signature: (Z)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_setUseSystemPicker + (JNIEnv*, jobject, jboolean); + + /* + * Class: dev_onvoid_webrtc_media_video_VideoDesktopSource + * Method: isSystemPickerAvailable + * Signature: ()Z + */ + JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_isSystemPickerAvailable + (JNIEnv*, jclass); + /* * Class: dev_onvoid_webrtc_media_video_VideoDesktopSource * Method: start diff --git a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h index bc8dc48d6..e8d706e8e 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h +++ b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h @@ -38,9 +38,18 @@ namespace jni void setFrameRate(const uint16_t frameRate); void setMaxFrameSize(webrtc::DesktopSize size); void setFocusSelectedSource(bool focus); + void setUseSystemPicker(bool usePicker); + + // Whether the system offers its own picker for screens and + // windows: macOS 14 and newer, and Wayland. + static bool isSystemPickerAvailable(); void start(); + // Stops capturing and waits for the capture thread to finish, + // unless called on that thread, as from an ended listener. void stop(); + // Stops capturing without waiting. + void requestStop(); void terminate(); // AdaptedVideoTrackSource implementation. @@ -58,11 +67,15 @@ namespace jni private: void capture(); void process(std::unique_ptr& frame); + bool onCaptureThread() const; private: uint16_t frameRate; std::atomic isCapturing; bool focusSelectedSource; + bool useSystemPicker; + // Set on the capture thread when the capturer gave up for good. + bool permanentError; webrtc::DesktopSize maxFrameSize; diff --git a/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp index e7249cb33..21d58090d 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp @@ -61,6 +61,23 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_set }); } +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_setUseSystemPicker +(JNIEnv * env, jobject caller, jboolean usePicker) +{ + jni::CallFromJava(env, [&]() { + jni::NativeLease videoSource(env, caller); + CHECK_HANDLE(videoSource); + + videoSource->setUseSystemPicker(static_cast(usePicker)); + }); +} + +JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_isSystemPickerAvailable +(JNIEnv * env, jclass caller) +{ + return static_cast(jni::VideoTrackDesktopSource::isSystemPickerAvailable()); +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_start (JNIEnv * env, jobject caller) { @@ -103,6 +120,10 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_dis return; } + // A track may keep the source alive, but not its capture, whose thread + // holds a reference of its own. Nothing waits for that thread here. + videoSource->requestStop(); + // A track may keep the source alive; it must not reach the processor // any more. videoSource->SetVideoProcessor(env, nullptr); diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index 448e90603..b0e1c2f8b 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -35,6 +35,7 @@ #if defined(WEBRTC_MAC) #include "media/video/desktop/macos/MacOSScreenCapturer.h" #include "media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h" +#include "modules/desktop_capture/mac/screen_capturer_sck.h" #endif #include @@ -42,11 +43,19 @@ namespace jni { + namespace + { + // The source whose capture thread this is, if any. + thread_local const VideoTrackDesktopSource * captureSource = nullptr; + } + VideoTrackDesktopSource::VideoTrackDesktopSource() : AdaptedVideoTrackSource(), frameRate(20), isCapturing(false), focusSelectedSource(true), + useSystemPicker(false), + permanentError(false), sourceState(kInitializing), sourceId(-1), sourceIsWindow(false) @@ -55,7 +64,25 @@ namespace jni VideoTrackDesktopSource::~VideoTrackDesktopSource() { - stop(); + isCapturing = false; + + if (captureThread.empty()) { + return; + } + + if (onCaptureThread()) { + // The capture thread let go of the last reference as it ended, for + // instance after an ended listener disposed the source and its + // track. A thread cannot join itself; a short-lived one joins it. + webrtc::PlatformThread::SpawnDetached( + [thread = std::move(captureThread)]() mutable { + thread.Finalize(); + }, + "VideoTrackDesktopJoin"); + } + else { + captureThread.Finalize(); + } } void VideoTrackDesktopSource::setSourceId(webrtc::DesktopCapturer::SourceId source, bool isWindow) @@ -79,14 +106,35 @@ namespace jni this->focusSelectedSource = focus; } + void VideoTrackDesktopSource::setUseSystemPicker(bool usePicker) + { + this->useSystemPicker = usePicker; + } + + bool VideoTrackDesktopSource::isSystemPickerAvailable() + { +#if defined(WEBRTC_MAC) + return webrtc::GenericCapturerSckWithPickerAvailable(); +#elif defined(WEBRTC_USE_PIPEWIRE) + return webrtc::DesktopCapturer::IsRunningUnderWayland(); +#else + return false; +#endif + } + void VideoTrackDesktopSource::start() { if (captureThread.empty()) { isCapturing = true; + permanentError = false; + + // The capture thread keeps the source alive until it ends, so + // that an ended listener on that thread may stop or dispose it. + webrtc::scoped_refptr self(this); captureThread = webrtc::PlatformThread::SpawnJoinable( - [&] { - capture(); + [self = std::move(self)] { + self->capture(); }, "VideoTrackDesktopThread", webrtc::ThreadAttributes().SetPriority(webrtc::ThreadPriority::kHigh)); @@ -97,10 +145,27 @@ namespace jni { if (!captureThread.empty()) { isCapturing = false; + + if (onCaptureThread()) { + // From an ended listener or a sink: the loop ends after this + // pass, and a later stop() or the destructor joins the thread. + return; + } + captureThread.Finalize(); } } + void VideoTrackDesktopSource::requestStop() + { + isCapturing = false; + } + + bool VideoTrackDesktopSource::onCaptureThread() const + { + return captureSource == this; + } + void VideoTrackDesktopSource::terminate() { // Notify the track that we are permanently done. @@ -134,7 +199,8 @@ namespace jni if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) { RTC_LOG(LS_ERROR) << "Permanent error capturing desktop frame. Stopping track."; - isCapturing = false; + permanentError = true; + isCapturing = false; } return; @@ -256,12 +322,23 @@ namespace jni void VideoTrackDesktopSource::capture() { + captureSource = this; + auto options = webrtc::DesktopCaptureOptions::CreateDefault(); // Enable desktop effects. options.set_disable_effects(false); #if defined(WEBRTC_MAC) options.set_allow_iosurface(true); + + if (useSystemPicker) { + // SCContentSharingPicker on macOS 14 and newer. ScreenCaptureKit + // draws the cursor: a picked window's frames start at the + // window's origin, where DesktopAndCursorComposer, working in + // desktop coordinates, would misplace it. + options.set_allow_sck_system_picker(true); + options.set_prefer_cursor_embedded(true); + } #endif #if defined(WEBRTC_WIN) options.set_allow_directx_capturer(true); @@ -277,13 +354,27 @@ namespace jni std::unique_ptr inner; + if (useSystemPicker) { + // The user chooses a screen or windows in the system's picker, so + // the source ID is not used. Without a picker the source ends + // rather than capture something the user did not choose. + inner = webrtc::DesktopCapturer::CreateGenericCapturer(options); + + if (!inner) { + RTC_LOG(LS_ERROR) << "VideoTrackDesktopSource: No system picker on this system."; + + terminate(); + return; + } + } + #if defined(WEBRTC_MAC) - if (sourceIsWindow) { + if (!inner && sourceIsWindow) { // Streams the window with ScreenCaptureKit on macOS 14 and newer, // instead of a CGWindowListCreateImage snapshot per frame. inner = CreateScreenCaptureKitWindowCapturer(options); } - else { + else if (!inner) { // A screen is captured with ScreenCaptureKit on macOS 14 and newer. // The CGDisplayStream capturer otherwise used gets new pixels only // when its stream calls back, and when that stops it keeps sending @@ -301,7 +392,9 @@ namespace jni // when PipeWire is disallowed, in which case we fall back to the // type-specific capturers below. The portal handles the actual source // selection, so sourceIsWindow is irrelevant on this path. - inner = webrtc::DesktopCapturer::CreateGenericCapturer(options); + if (!inner) { + inner = webrtc::DesktopCapturer::CreateGenericCapturer(options); + } #endif if (!inner) { @@ -366,5 +459,12 @@ namespace jni // Frees what a burst of kept frames left behind. Buffers still in use // stay with their frames. bufferPool.Release(); + + if (permanentError) { + // The capturer gave up for good: the window was closed, or the + // user cancelled the system picker or stopped sharing. The ended + // listeners run on this thread. + terminate(); + } } } \ No newline at end of file diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java index b08006dbc..a2074d736 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java @@ -58,10 +58,52 @@ public void setFrameRate(int frameRate) { */ public native void setMaxFrameSize(int width, int height); + /** + * Lets the user choose what to share in the operating system's own + * picker when the source starts, instead of capturing the screen or window + * set with {@link #setSourceId(long, boolean)}. Off by default; read when + * the capture starts. + *

+ * On macOS 14 and newer this is ScreenCaptureKit's content sharing + * picker, which offers a screen or one or more windows. Under Wayland the + * desktop portal's picker is used with or without this setting. Where + * {@link #isSystemPickerAvailable()} is false, the source ends instead of + * capturing. + *

+ * {@link #start()} shows the picker and returns at once; no frames arrive + * until the user has chosen. If the user cancels the picker, or stops + * sharing from the menu bar, the source ends and its tracks call their + * {@link dev.onvoid.webrtc.media.MediaStreamTrackEndedListener}s. Show the + * picker for one source at a time: sources waiting on it at the same time + * all receive the user's choice. + * + * @param usePicker True to show the system picker when the source starts. + */ + public native void setUseSystemPicker(boolean usePicker); + + /** + * Whether this system offers a picker for + * {@link #setUseSystemPicker(boolean)}: macOS 14 and newer, and Linux + * under Wayland. + * + * @return True if a system picker is available. + */ + public static native boolean isSystemPickerAvailable(); + public native void start(); + /** + * Stops capturing and waits for the capture to finish. Called from a + * listener that the capture invokes, such as an ended listener, it does + * not wait. + */ public native void stop(); + /** + * Stops capturing, without waiting for the capture to finish, and + * releases the source. A track created from the source keeps it, but + * receives no more frames. + */ public void dispose() { detachVideoProcessor(); disposeInternal(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java index 583a9a75d..c3a94cdf6 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java @@ -251,7 +251,7 @@ void windowFramesFollowWindow() throws Exception { } @Test - void closedWindowStopsCapture() throws Exception { + void closedWindowEndsTrack() throws Exception { assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); @@ -261,22 +261,153 @@ void closedWindowStopsCapture() throws Exception { assumeTrue(source != null, "The window is not listed under its title"); try (WindowCapture capture = new WindowCapture(source)) { + CountDownLatch ended = new CountDownLatch(1); + + capture.track.addTrackEndedListener(track -> ended.countDown()); + assumeTrue(capture.await(frame -> true, 5000) != null, "The window delivered no frames"); window.closeWindow(); - // Capture stops within a frame or two of the window closing. - Thread.sleep(1000); + assertTrue(ended.await(5, TimeUnit.SECONDS), "The track did not end"); + assertEquals(MediaStreamTrackState.ENDED, capture.track.getState()); int frames = capture.frames(); - Thread.sleep(1000); + Thread.sleep(500); assertEquals(frames, capture.frames(), "Frames kept coming after the window closed"); } } } + @Test + void endedListenerMayStopAndDispose() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + try (TestWindowProcess window = TestWindowProcess.open(400, 300)) { + DesktopSource source = window.find(); + + assumeTrue(source != null, "The window is not listed under its title"); + + try (WindowCapture capture = new WindowCapture(source)) { + CountDownLatch ended = new CountDownLatch(1); + + // The listener runs on the capture thread, which stop() must + // not wait for there (issue #186). + capture.track.addTrackEndedListener(track -> { + capture.stopAndDisposeSource(); + ended.countDown(); + }); + + assumeTrue(capture.await(frame -> true, 5000) != null, "The window delivered no frames"); + + window.closeWindow(); + + assertTrue(ended.await(5, TimeUnit.SECONDS), "The listener did not finish"); + assertEquals(MediaStreamTrackState.ENDED, capture.track.getState()); + } + } + } + + @Test + void disposeEndsCapture() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + try (TestWindowProcess window = TestWindowProcess.open(400, 300)) { + DesktopSource desktopSource = window.find(); + + assumeTrue(desktopSource != null, "The window is not listed under its title"); + + VideoDesktopSource source = new VideoDesktopSource(); + source.setSourceId(desktopSource.id, true); + source.setFrameRate(10); + + VideoTrack track = factory.createVideoTrack("window", source); + AtomicInteger frames = new AtomicInteger(); + + VideoTrackSink sink = frame -> { + frame.release(); + frames.incrementAndGet(); + }; + + try { + track.addSink(sink); + source.start(); + + for (int i = 0; i < 50 && frames.get() == 0; i++) { + Thread.sleep(100); + } + + assumeTrue(frames.get() > 0, "The window delivered no frames"); + + // The track keeps the source, but disposing it ends its capture. + source.dispose(); + + Thread.sleep(500); + + int count = frames.get(); + + Thread.sleep(1000); + + assertEquals(count, frames.get(), "Frames kept coming after the source was disposed"); + } + finally { + track.removeSink(sink); + track.dispose(); + source.dispose(); + } + } + } + + @Test + void systemPickerAvailability() { + boolean expected; + + if (OS.contains("mac")) { + expected = Integer.parseInt(System.getProperty("os.version").split("\\.")[0]) >= 14; + } + else if (OS.contains("linux")) { + String session = System.getenv("XDG_SESSION_TYPE"); + + expected = session != null && session.startsWith("wayland") && System.getenv("WAYLAND_DISPLAY") != null; + } + else { + expected = false; + } + + assertEquals(expected, VideoDesktopSource.isSystemPickerAvailable()); + } + + @Test + void endsWithoutPicker() throws Exception { + // With a picker, starting would wait for the user. + assumeFalse(VideoDesktopSource.isSystemPickerAvailable(), "This system has a picker"); + + VideoDesktopSource source = new VideoDesktopSource(); + source.setUseSystemPicker(true); + + VideoTrack track = factory.createVideoTrack("picker", source); + AtomicInteger ended = new AtomicInteger(); + + track.addTrackEndedListener(t -> ended.incrementAndGet()); + + try { + source.start(); + // Waits for the capture thread, which has given up by then. + source.stop(); + + assertEquals(MediaStreamTrackState.ENDED, track.getState()); + assertEquals(1, ended.get()); + } + finally { + track.dispose(); + source.dispose(); + } + } + @Test void ownWindowCapturedUntilClosed() throws Exception { assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); @@ -514,6 +645,8 @@ private final class WindowCapture implements AutoCloseable { private final AtomicReference latest = new AtomicReference<>(); + private volatile boolean sourceDisposed; + WindowCapture(DesktopSource window) { source.setSourceId(window.id, true); @@ -553,12 +686,28 @@ int frames() { return frames.get(); } + /** + * Stops and disposes the source, as an application might do in an + * ended listener. + */ + void stopAndDisposeSource() { + source.stop(); + source.dispose(); + sourceDisposed = true; + } + @Override public void close() { - source.stop(); + if (!sourceDisposed) { + source.stop(); + } + track.removeSink(sink); track.dispose(); - source.dispose(); + + if (!sourceDisposed) { + source.dispose(); + } } } From 9e8b774518d54271c15a8070ca6a280134746c9c Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 13 Nov 2026 20:06:53 +0100 Subject: [PATCH 2/3] feat: offer the system picker on ScreenCapturer and WindowCapturer DesktopCapturer.setUseSystemPicker(true), set before start(), makes start() show the operating system's own picker instead of capturing the source chosen with selectSource(): ScreenCaptureKit's SCContentSharingPicker on macOS 14 and newer, the desktop portal under Wayland, as VideoDesktopSource already does. Until the user has chosen, captureFrame() reports ERROR_TEMPORARY; a cancelled picker or stopped sharing reports ERROR_PERMANENT, as does every frame where isSystemPickerAvailable() is false. Setting it after start() throws IllegalStateException. Changing the setting replaces the native capturer, keeping the maximum frame rate. VideoDesktopSource now asks DesktopCapturer whether a picker is available, so both answer alike. --- docs/guide/video/desktop-capture.md | 2 +- docs/tools/desktop/screen-capturer.md | 32 ++++++ docs/tools/desktop/window-capturer.md | 6 ++ .../main/cpp/include/JNI_DesktopCapturer.h | 16 +++ .../media/video/desktop/DesktopCapturer.h | 19 ++++ .../src/main/cpp/src/JNI_DesktopCapturer.cpp | 19 ++++ .../media/video/VideoTrackDesktopSource.cpp | 9 +- .../media/video/desktop/DesktopCapturer.cpp | 100 +++++++++++++++--- .../media/video/desktop/DesktopCapturer.java | 36 +++++++ .../video/desktop/DesktopCapturerTests.java | 65 ++++++++++++ 10 files changed, 282 insertions(+), 22 deletions(-) diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index f3d1f26ad..d2d68ff7b 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -89,7 +89,7 @@ else { `start()` shows the picker and returns at once. No frames arrive until the user has made a choice. The user can change it later from the macOS menu bar, and the frames follow. If the user cancels the picker, or stops sharing from the menu bar, the source ends (see [Handling the End of Capture](#handling-the-end-of-capture)). Where `isSystemPickerAvailable()` is false, a source set to use the picker ends right away instead of capturing. -On macOS the picker offers a whole screen or one or more windows. Show it for one source at a time: sources waiting on it at the same time all receive the user's choice. The [`DesktopPickerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java) shows the picker and reports the frames it receives. +On macOS the picker offers a whole screen or one or more windows. Show it for one source at a time: sources waiting on it at the same time all receive the user's choice. To take frames yourself rather than through a track, `ScreenCapturer` and `WindowCapturer` offer the same picker (see [Screen Capturer](/tools/desktop/screen-capturer#letting-the-user-choose-in-the-system-picker)). The [`DesktopPickerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopPickerExample.java) shows the picker and reports the frames it receives. ### Creating a Video Track with the Desktop Source diff --git a/docs/tools/desktop/screen-capturer.md b/docs/tools/desktop/screen-capturer.md index 67b9219b1..52b318409 100644 --- a/docs/tools/desktop/screen-capturer.md +++ b/docs/tools/desktop/screen-capturer.md @@ -16,6 +16,8 @@ Key methods (inherited from `DesktopCapturer`): - `void selectSource(DesktopSource source)` – choose the screen to capture - `void setFocusSelectedSource(boolean focus)` – try to focus the selected source during capture - `void setMaxFrameRate(int maxFrameRate)` – cap the capture FPS +- `void setUseSystemPicker(boolean usePicker)` – let the user choose a screen or windows in the operating system's picker, shown by `start`, instead of `selectSource`; set it before `start`, a later call throws `IllegalStateException` +- `static boolean isSystemPickerAvailable()` – whether the system has such a picker: macOS 14 and newer, and Linux under Wayland - `void start(DesktopCaptureCallback callback)` – begin capturing; frames are delivered via callback. A capturer can be started once, a second call throws `IllegalStateException` - `void captureFrame()` – request a single frame (manual capture) after `start`; without a capture backend, as on Linux without a display, the callback gets `ERROR_PERMANENT` - `void dispose()` – release resources @@ -71,6 +73,36 @@ finally { } ``` +## Letting the user choose in the system picker + +Instead of selecting a source, a capturer can show the operating system's own picker when it starts. On macOS 14 and newer that is ScreenCaptureKit's content sharing picker, which offers a whole screen or one or more windows, for a `ScreenCapturer` and a `WindowCapturer` alike. Under Wayland it is the desktop portal's picker: + +```java +ScreenCapturer capturer = new ScreenCapturer(); +try { + if (DesktopCapturer.isSystemPickerAvailable()) { + // The picker chooses the source; no selectSource() is needed. + capturer.setUseSystemPicker(true); + } + + capturer.start((result, frame) -> { + if (result == DesktopCapturer.Result.SUCCESS && frame != null) { + // process VideoFrame + } + else if (result == DesktopCapturer.Result.ERROR_PERMANENT) { + // the picker was cancelled, or sharing stopped + } + }); + + // Call captureFrame() periodically +} +finally { + capturer.dispose(); +} +``` + +Set it before `start`, which shows the picker and returns at once. Until the user has chosen, `captureFrame()` reports `ERROR_TEMPORARY`. If the user cancels the picker, or stops sharing from the macOS menu bar, it reports `ERROR_PERMANENT`. Where `isSystemPickerAvailable()` is false, it reports `ERROR_PERMANENT` from the first frame on, instead of capturing something the user did not choose. Show the picker for one capturer at a time: capturers waiting on it at the same time all receive the user's choice. + ## Using with VideoDesktopSource to create a VideoTrack For most WebRTC pipelines, you will create a `VideoDesktopSource` and set the selected screen ID: diff --git a/docs/tools/desktop/window-capturer.md b/docs/tools/desktop/window-capturer.md index 6f533b5c6..bf383a372 100644 --- a/docs/tools/desktop/window-capturer.md +++ b/docs/tools/desktop/window-capturer.md @@ -16,6 +16,8 @@ Key methods (inherited from `DesktopCapturer`): - `void selectSource(DesktopSource source)` – choose which window to capture - `void setFocusSelectedSource(boolean focus)` – attempt to focus/raise the selected window - `void setMaxFrameRate(int maxFrameRate)` – limit the capture frame rate +- `void setUseSystemPicker(boolean usePicker)` – let the user choose a screen or windows in the operating system's picker, shown by `start`, instead of `selectSource`; set it before `start`, a later call throws `IllegalStateException` +- `static boolean isSystemPickerAvailable()` – whether the system has such a picker: macOS 14 and newer, and Linux under Wayland - `void start(DesktopCaptureCallback callback)` – start capture, frames via callback. A capturer can be started once, a second call throws `IllegalStateException` - `void captureFrame()` – request a single capture after `start`; without a capture backend, as on Linux without a display, the callback gets `ERROR_PERMANENT` - `void dispose()` – release resources @@ -68,6 +70,10 @@ finally { } ``` +## Letting the user choose in the system picker + +With `setUseSystemPicker(true)` before `start`, the capturer shows the operating system's picker instead of capturing a selected window. It works the same way for a `WindowCapturer` as for a `ScreenCapturer`; see [Letting the user choose in the system picker](/tools/desktop/screen-capturer#letting-the-user-choose-in-the-system-picker). + ## Using with VideoDesktopSource to create a VideoTrack ```java diff --git a/webrtc-jni/src/main/cpp/include/JNI_DesktopCapturer.h b/webrtc-jni/src/main/cpp/include/JNI_DesktopCapturer.h index a9e46aeb0..9e1d4355f 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_DesktopCapturer.h +++ b/webrtc-jni/src/main/cpp/include/JNI_DesktopCapturer.h @@ -47,6 +47,22 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_setMaxFrameRate (JNIEnv*, jobject, jint); + /* + * Class: dev_onvoid_webrtc_media_video_desktop_DesktopCapturer + * Method: setUseSystemPicker + * Signature: (Z)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_setUseSystemPicker + (JNIEnv *, jobject, jboolean); + + /* + * Class: dev_onvoid_webrtc_media_video_desktop_DesktopCapturer + * Method: isSystemPickerAvailable + * Signature: ()Z + */ + JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_isSystemPickerAvailable + (JNIEnv *, jclass); + /* * Class: dev_onvoid_webrtc_media_video_desktop_DesktopCapturer * Method: start diff --git a/webrtc-jni/src/main/cpp/include/media/video/desktop/DesktopCapturer.h b/webrtc-jni/src/main/cpp/include/media/video/desktop/DesktopCapturer.h index d45cf9fe4..d60e23c72 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/desktop/DesktopCapturer.h +++ b/webrtc-jni/src/main/cpp/include/media/video/desktop/DesktopCapturer.h @@ -55,10 +55,24 @@ namespace jni bool SelectSource(webrtc::DesktopCapturer::SourceId id); bool FocusOnSelectedSource(); void setFocusSelectedSource(bool focus); + + // Lets the user choose a screen or windows in the system's picker, + // shown by Start(), instead of a source selected here. Replaces the + // capturer, so a selected source is forgotten. Returns false if + // capture has been started. + bool setUseSystemPicker(bool usePicker); + + // Whether the system offers its own picker for screens and + // windows: macOS 14 and newer, and Wayland. + static bool isSystemPickerAvailable(); + bool IsOccluded(const webrtc::DesktopVector & pos); bool isWindowCapturer() const; protected: + // Creates the capturer for the current settings. + void createCapturer(); + std::mutex startMutex; // Declared before the capturer, which uses it until destroyed. @@ -68,6 +82,11 @@ namespace jni bool focusSelectedSource; + bool useSystemPicker; + + // Zero until set. + uint32_t maxFrameRate; + const bool windowCapturer; #if defined(WEBRTC_WIN) diff --git a/webrtc-jni/src/main/cpp/src/JNI_DesktopCapturer.cpp b/webrtc-jni/src/main/cpp/src/JNI_DesktopCapturer.cpp index 3660e1b0f..1d510a9c9 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_DesktopCapturer.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_DesktopCapturer.cpp @@ -112,6 +112,25 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapture }); } +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_setUseSystemPicker +(JNIEnv * env, jobject caller, jboolean usePicker) +{ + jni::CallFromJava(env, [&]() { + jni::NativeLease capturer(env, caller); + CHECK_HANDLE(capturer); + + if (!capturer->setUseSystemPicker(static_cast(usePicker))) { + ThrowJavaException(env, "java/lang/IllegalStateException", "Capture has been started already"); + } + }); +} + +JNIEXPORT jboolean JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_isSystemPickerAvailable +(JNIEnv * env, jclass caller) +{ + return static_cast(jni::DesktopCapturer::isSystemPickerAvailable()); +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_desktop_DesktopCapturer_start (JNIEnv * env, jobject caller, jobject jcallback) { diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index b0e1c2f8b..32a9175a7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -15,6 +15,7 @@ */ #include "media/video/VideoTrackDesktopSource.h" +#include "media/video/desktop/DesktopCapturer.h" #include "media/video/desktop/WindowFrameCrop.h" #include "Exception.h" @@ -113,13 +114,7 @@ namespace jni bool VideoTrackDesktopSource::isSystemPickerAvailable() { -#if defined(WEBRTC_MAC) - return webrtc::GenericCapturerSckWithPickerAvailable(); -#elif defined(WEBRTC_USE_PIPEWIRE) - return webrtc::DesktopCapturer::IsRunningUnderWayland(); -#else - return false; -#endif + return DesktopCapturer::isSystemPickerAvailable(); } void VideoTrackDesktopSource::start() diff --git a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp index edd91dcc2..854a53b0f 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp @@ -25,20 +25,39 @@ #if defined(WEBRTC_MAC) #include "media/video/desktop/macos/MacOSScreenCapturer.h" #include "media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h" +#include "modules/desktop_capture/mac/screen_capturer_sck.h" #endif namespace jni { DesktopCapturer::DesktopCapturer(bool screenCapturer) : focusSelectedSource(false), + useSystemPicker(false), + maxFrameRate(0), windowCapturer(!screenCapturer) { + createCapturer(); + } + + void DesktopCapturer::createCapturer() + { + capturer.reset(); + auto options = webrtc::DesktopCaptureOptions::CreateDefault(); // Enable desktop effects. options.set_disable_effects(false); #if defined(WEBRTC_MAC) options.set_allow_iosurface(true); + + if (useSystemPicker) { + // SCContentSharingPicker on macOS 14 and newer. ScreenCaptureKit + // draws the cursor: a picked window's frames start at the + // window's origin, where DesktopAndCursorComposer, working in + // desktop coordinates, would misplace it. + options.set_allow_sck_system_picker(true); + options.set_prefer_cursor_embedded(true); + } #endif #if defined(WEBRTC_WIN) options.set_allow_directx_capturer(true); @@ -54,33 +73,53 @@ namespace jni std::unique_ptr inner; + if (useSystemPicker) { + // The user chooses a screen or windows in the system's picker, + // which Start() shows. Under Wayland that is the portal's picker + // for screens and windows alike. Without a picker there is no + // capturer, rather than one capturing something the user did not + // choose. + inner = webrtc::DesktopCapturer::CreateGenericCapturer(options); + + if (!inner) { + RTC_LOG(LS_ERROR) << "DesktopCapturer: No system picker on this system."; + return; + } + } + #if defined(WEBRTC_MAC) - // Both list their sources as WebRTC's capturers do, and capture with - // ScreenCaptureKit on macOS 14 and newer. - inner = screenCapturer - ? CreateMacOSScreenCapturer(options) - : CreateScreenCaptureKitWindowCapturer(options); + if (!inner) { + // Both list their sources as WebRTC's capturers do, and capture with + // ScreenCaptureKit on macOS 14 and newer. + inner = windowCapturer + ? CreateScreenCaptureKitWindowCapturer(options) + : CreateMacOSScreenCapturer(options); + } #endif if (!inner) { - inner = screenCapturer - ? webrtc::DesktopCapturer::CreateScreenCapturer(options) - : webrtc::DesktopCapturer::CreateWindowCapturer(options); + inner = windowCapturer + ? webrtc::DesktopCapturer::CreateWindowCapturer(options) + : webrtc::DesktopCapturer::CreateScreenCapturer(options); } if (!inner) { - RTC_LOG(LS_ERROR) << (screenCapturer - ? "DesktopCapturer: CreateScreenCapturer returned null. " + RTC_LOG(LS_ERROR) << (windowCapturer + ? "DesktopCapturer: CreateWindowCapturer returned null. " + "Possible causes: missing X11/XFIXES extension or snap confinement." + : "DesktopCapturer: CreateScreenCapturer returned null. " "Possible causes: missing X11/XDAMAGE extension, Wayland " - "session without PipeWire support, or snap confinement." - : "DesktopCapturer: CreateWindowCapturer returned null. " - "Possible causes: missing X11/XFIXES extension or snap confinement."); + "session without PipeWire support, or snap confinement."); // Leave capturer as nullptr: GetSourceList returns false, and // CaptureFrame reports ERROR_PERMANENT. return; } capturer = std::make_unique(std::move(inner), options); + + if (maxFrameRate > 0) { + capturer->SetMaxFrameRate(maxFrameRate); + } } DesktopCapturer::~DesktopCapturer() @@ -111,6 +150,9 @@ namespace jni void DesktopCapturer::SetMaxFrameRate(uint32_t max_frame_rate) { + // Kept for a capturer replaced by setUseSystemPicker(). + maxFrameRate = max_frame_rate; + if (!capturer) return; capturer->SetMaxFrameRate(max_frame_rate); } @@ -186,8 +228,38 @@ namespace jni return capturer->IsOccluded(pos); } + bool DesktopCapturer::setUseSystemPicker(bool usePicker) + { + std::lock_guard lock(startMutex); + + if (captureCallback) { + return false; + } + + if (useSystemPicker != usePicker) { + useSystemPicker = usePicker; + + createCapturer(); + } + + return true; + } + + bool DesktopCapturer::isSystemPickerAvailable() + { +#if defined(WEBRTC_MAC) + return webrtc::GenericCapturerSckWithPickerAvailable(); +#elif defined(WEBRTC_USE_PIPEWIRE) + return webrtc::DesktopCapturer::IsRunningUnderWayland(); +#else + return false; +#endif + } + bool DesktopCapturer::isWindowCapturer() const { - return windowCapturer; + // A picked window is captured by ScreenCaptureKit or the portal, + // whose frames have no window border to crop. + return windowCapturer && !useSystemPicker; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturer.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturer.java index 10ecbd66d..9ca0cb763 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturer.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturer.java @@ -101,6 +101,42 @@ public enum Result { */ public native void setMaxFrameRate(int maxFrameRate); + /** + * Lets the user choose what to capture in the operating system's own + * picker, instead of the source set with {@link #selectSource}. Off by + * default. Set it before {@link #start}; changing it replaces the native + * capturer, which forgets a selected source. + *

+ * On macOS 14 and newer this is ScreenCaptureKit's content sharing + * picker, which offers a screen or one or more windows, for a + * {@link ScreenCapturer} and a {@link WindowCapturer} alike. Under + * Wayland it is the desktop portal's picker for screens and windows. + * Where {@link #isSystemPickerAvailable()} is false, every + * {@link #captureFrame()} reports {@link Result#ERROR_PERMANENT} instead + * of capturing something the user did not choose. + *

+ * {@link #start} shows the picker and returns at once. Until the user + * has chosen, {@link #captureFrame()} reports {@link Result#ERROR_TEMPORARY}. + * If the user cancels the picker, or stops sharing from the menu bar, it + * reports {@link Result#ERROR_PERMANENT}. Show the picker for one capturer + * at a time: capturers waiting on it at the same time all receive the + * user's choice. + * + * @param usePicker True to show the system picker when capture starts. + * + * @throws IllegalStateException If the capturer has been started. + */ + public native void setUseSystemPicker(boolean usePicker); + + /** + * Whether this system offers a picker for + * {@link #setUseSystemPicker(boolean)}: macOS 14 and newer, and Linux + * under Wayland. + * + * @return True if a system picker is available. + */ + public static native boolean isSystemPickerAvailable(); + /** * Starts the desktop capture process with the provided callback. A capturer * can be started once and keeps its callback until it is disposed. diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java index 683007845..5b0bb28b9 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java @@ -24,6 +24,7 @@ import dev.onvoid.webrtc.TestBase; import dev.onvoid.webrtc.media.video.I420Buffer; +import dev.onvoid.webrtc.media.video.VideoDesktopSource; import java.awt.Color; import java.awt.EventQueue; @@ -89,6 +90,70 @@ void missingBackendReportsPermanentError() { } } + @Test + void systemPickerAvailability() { + // VideoDesktopSourceTests checks which systems have one. + assertEquals(VideoDesktopSource.isSystemPickerAvailable(), DesktopCapturer.isSystemPickerAvailable()); + } + + @Test + void pickerAfterStartIsRejected() { + assumeNoPortalDialog(); + + ScreenCapturer capturer = new ScreenCapturer(); + + try { + capturer.start((result, frame) -> { }); + + assertThrows(IllegalStateException.class, () -> capturer.setUseSystemPicker(true)); + } + finally { + capturer.dispose(); + } + } + + @Test + void noPickerReportsPermanentError() { + // With a picker, starting would wait for the user. + assumeFalse(DesktopCapturer.isSystemPickerAvailable(), "This system has a picker"); + + ScreenCapturer capturer = new ScreenCapturer(); + + try { + List results = new ArrayList<>(); + + capturer.setUseSystemPicker(true); + capturer.start((result, frame) -> results.add(result)); + capturer.captureFrame(); + + assertEquals(List.of(DesktopCapturer.Result.ERROR_PERMANENT), results); + } + finally { + capturer.dispose(); + } + } + + @Test + void pickerCanBeTurnedOff() { + ScreenCapturer capturer = new ScreenCapturer(); + + try { + assumeFalse(hasNoBackend(capturer), "No capture backend"); + + List screens = capturer.getDesktopSources(); + + // Each replaces the native capturer, which shows no picker + // before it starts. + capturer.setUseSystemPicker(true); + capturer.setUseSystemPicker(false); + + assertEquals(screens, capturer.getDesktopSources()); + } + finally { + capturer.dispose(); + } + } + @Test void windowCapturerFollowsWindowUntilClosed() throws Exception { assumeTrue(System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac"), From f8a61dee010fc251d57d63bcd881ae6b6527dcfc Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 13 Nov 2026 20:14:26 +0100 Subject: [PATCH 3/3] docs: add a Swing example for the DesktopCapturer system picker DesktopCapturerPickerExample starts a ScreenCapturer set to use the system picker, calls captureFrame() at the frame rate on the one thread that uses the capturer, and shows the frames in a VideoView. Its status line tells a pending choice from a cancelled picker or stopped sharing. --- docs/guide/examples.md | 14 + .../DesktopCapturerPickerExample.java | 242 ++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopCapturerPickerExample.java diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 7828bf2ed..beefa694c 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -157,6 +157,20 @@ The [`DesktopPickerExample`](https://github.com/devopvoid/webrtc-java/blob/maste - Showing the picker by starting a `VideoDesktopSource` - Reacting to the end of the track when the picker is cancelled or sharing stops +## Desktop Capturer Picker + +The [`DesktopCapturerPickerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopCapturerPickerExample.java) is a small Swing application that takes frames from a `ScreenCapturer` without a video track, after the user has chosen a screen or window in the operating system's picker. + +**Key features demonstrated:** +- Setting `setUseSystemPicker(true)` before `start()`, which shows the picker +- Calling `captureFrame()` at the frame rate, on the one thread that uses the capturer +- Showing the frames in a Swing `VideoView` +- Telling a pending choice (`ERROR_TEMPORARY`) from a cancelled picker or stopped sharing (`ERROR_PERMANENT`) + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.DesktopCapturerPickerExample" +``` + ## Media File The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopCapturerPickerExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopCapturerPickerExample.java new file mode 100644 index 000000000..9007493a9 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopCapturerPickerExample.java @@ -0,0 +1,242 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Dimension; +import java.awt.FlowLayout; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.swing.BorderFactory; +import javax.swing.JButton; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.SwingUtilities; + +import dev.onvoid.webrtc.client.swing.VideoView; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.desktop.DesktopCapturer; +import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer; + +/** + * Swing example taking frames from a {@link DesktopCapturer} whose source the + * user chooses in the operating system's own picker, without a video track. + *

+ * "Choose" starts a {@link ScreenCapturer} set to use the system picker: on + * macOS 14 and newer, ScreenCaptureKit's content sharing picker, which offers + * a screen or one or more windows; under Wayland, the desktop portal's + * picker. The window shows the captured frames and the last capture result. + * Cancelling the picker, or stopping the sharing from the macOS menu bar, ends + * the capture with {@link DesktopCapturer.Result#ERROR_PERMANENT}. + *

+ * A capturer is used from one thread only, here a single scheduled thread + * that starts it, calls {@link DesktopCapturer#captureFrame()} at the frame + * rate and disposes of it. + *

+ * Run it with: + *

+ * mvn -pl webrtc-examples exec:java -Dexec.mainClass=dev.onvoid.webrtc.examples.DesktopCapturerPickerExample
+ * 
+ * + * @author Alex Andres + */ +public class DesktopCapturerPickerExample { + + private static final int FRAME_RATE = 30; + + private final ScheduledExecutorService captureThread = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "desktop-capture"); + thread.setDaemon(true); + return thread; + }); + + private final VideoView view = new VideoView(); + private final JLabel status = new JLabel(); + private final JButton chooseButton = new JButton("Choose…"); + private final JButton stopButton = new JButton("Stop"); + private final JFrame window = new JFrame("Desktop Capturer Picker"); + + // Used on the capture thread only. + private DesktopCapturer capturer; + private DesktopCapturer.Result lastResult; + private boolean ended; + private long frames; + + + public static void main(String[] args) { + SwingUtilities.invokeLater(() -> new DesktopCapturerPickerExample().show()); + } + + private void show() { + view.setBackground(Color.DARK_GRAY); + view.setPreferredSize(new Dimension(960, 540)); + + status.setBorder(BorderFactory.createEmptyBorder(0, 8, 0, 8)); + + chooseButton.addActionListener(e -> startCapture()); + stopButton.addActionListener(e -> stopCapture("Stopped.")); + stopButton.setEnabled(false); + + JPanel controls = new JPanel(new FlowLayout(FlowLayout.LEFT)); + controls.add(chooseButton); + controls.add(stopButton); + + JPanel bottom = new JPanel(new BorderLayout()); + bottom.add(controls, BorderLayout.WEST); + bottom.add(status, BorderLayout.CENTER); + + window.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + window.addWindowListener(new WindowAdapter() { + + @Override + public void windowClosed(WindowEvent e) { + captureThread.execute(() -> disposeCapturer()); + captureThread.shutdown(); + } + }); + window.getContentPane().add(view, BorderLayout.CENTER); + window.getContentPane().add(bottom, BorderLayout.SOUTH); + window.pack(); + window.setLocationRelativeTo(null); + window.setVisible(true); + + if (DesktopCapturer.isSystemPickerAvailable()) { + setStatus("Press \"Choose…\" to pick a screen or window."); + } + else { + chooseButton.setEnabled(false); + setStatus("This system offers no picker for screens and windows."); + } + + // Captures while a capturer is running. + captureThread.scheduleAtFixedRate(this::captureFrame, 0, 1000 / FRAME_RATE, TimeUnit.MILLISECONDS); + } + + private void startCapture() { + chooseButton.setEnabled(false); + stopButton.setEnabled(true); + view.clear(); + + captureThread.execute(() -> { + disposeCapturer(); + + lastResult = null; + ended = false; + frames = 0; + + capturer = new ScreenCapturer(); + capturer.setMaxFrameRate(FRAME_RATE); + // Set before start(). No selectSource() is needed: the user + // chooses in the picker, which start() shows. + capturer.setUseSystemPicker(true); + capturer.start(this::onCaptureResult); + + setStatus("Choose a screen or window in the picker."); + }); + } + + private void stopCapture(String message) { + captureThread.execute(() -> { + disposeCapturer(); + setStatus(message); + }); + + chooseButton.setEnabled(true); + stopButton.setEnabled(false); + } + + private void captureFrame() { + if (capturer == null) { + return; + } + + try { + capturer.captureFrame(); + } + catch (RuntimeException e) { + ended = true; + setStatus("Capture failed: " + e.getMessage()); + } + + if (ended) { + // Disposed of here, not in the callback that reported the end. + disposeCapturer(); + + SwingUtilities.invokeLater(() -> { + chooseButton.setEnabled(true); + stopButton.setEnabled(false); + }); + } + } + + /** + * Called on the capture thread from within {@link DesktopCapturer#captureFrame()}. + */ + private void onCaptureResult(DesktopCapturer.Result result, VideoFrame frame) { + try { + switch (result) { + case SUCCESS: + if (frame != null) { + view.accept(frame); + frames++; + + setStatus(String.format("Capturing: %d frames, %dx%d", frames, + frame.buffer.getWidth(), frame.buffer.getHeight())); + } + break; + + case ERROR_TEMPORARY: + // No frame yet, while the picker waits for the user. + if (lastResult != DesktopCapturer.Result.ERROR_TEMPORARY && frames == 0) { + setStatus("Waiting for a choice in the picker…"); + } + break; + + default: + ended = true; + setStatus(frames == 0 + ? "The picker was cancelled." + : "Sharing stopped after " + frames + " frames."); + break; + } + + lastResult = result; + } + finally { + if (frame != null) { + frame.release(); + } + } + } + + private void disposeCapturer() { + if (capturer != null) { + capturer.dispose(); + capturer = null; + } + } + + private void setStatus(String text) { + SwingUtilities.invokeLater(() -> status.setText(text)); + } +}