diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index 42dc29314..9712ed7be 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -184,6 +184,12 @@ screenCapturer.dispose(); windowCapturer.dispose(); ``` +### Capturing on macOS + +On macOS 14 and newer, screens and windows are captured with ScreenCaptureKit; older releases use the earlier Core Graphics capture. Both need the app to have Screen Recording permission (System Settings → Privacy & Security → Screen & System Audio Recording). Without it, the windows of other apps are listed without their titles and their content cannot be captured. Windows of the app's own process need no permission, and are captured with Core Graphics. + +While a captured window is minimized or hidden, its last image repeats. When it is closed, capture stops. + ## Conclusion This guide has demonstrated how to set up a WebRTC peer connection with desktop video capture capabilities. diff --git a/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/MacOSScreenCapturer.h b/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/MacOSScreenCapturer.h new file mode 100644 index 000000000..125e80dfc --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/MacOSScreenCapturer.h @@ -0,0 +1,38 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_DESKTOP_MACOS_SCREEN_CAPTURER_H_ +#define JNI_WEBRTC_MEDIA_DESKTOP_MACOS_SCREEN_CAPTURER_H_ + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" + +#include + +namespace jni +{ + // Creates a capturer for one screen, or for the whole desktop, that picks + // its backend when started. A single screen is captured with + // ScreenCaptureKit on macOS 14 and newer, which draws the cursor itself. + // The whole desktop, which ScreenCaptureKit cannot capture, and every + // screen on older macOS, are captured with WebRTC's CGDisplayStream + // capturer. That capturer also lists the screens, since ScreenCaptureKit + // lists none without its system picker; both name a screen by its + // CGDirectDisplayID. + std::unique_ptr CreateMacOSScreenCapturer(const webrtc::DesktopCaptureOptions & options); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h b/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h new file mode 100644 index 000000000..091929424 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_DESKTOP_MACOS_SCREEN_CAPTURE_KIT_WINDOW_CAPTURER_H_ +#define JNI_WEBRTC_MEDIA_DESKTOP_MACOS_SCREEN_CAPTURE_KIT_WINDOW_CAPTURER_H_ + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" + +#include + +namespace jni +{ + // Creates a capturer for one window, selected by its CGWindowID, that + // streams it with ScreenCaptureKit instead of taking a + // CGWindowListCreateImage snapshot for every frame, as WebRTC's window + // capturer does. Windows of this process, and windows ScreenCaptureKit + // cannot capture, for example without screen recording permission, are + // captured with WebRTC's window capturer instead. Returns null before + // macOS 14. + std::unique_ptr CreateScreenCaptureKitWindowCapturer(const webrtc::DesktopCaptureOptions & options); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index f13a2dc85..448e90603 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -32,6 +32,11 @@ #include "modules/desktop_capture/desktop_and_cursor_composer.h" #include "modules/desktop_capture/mouse_cursor_monitor.h" +#if defined(WEBRTC_MAC) +#include "media/video/desktop/macos/MacOSScreenCapturer.h" +#include "media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h" +#endif + #include #include @@ -257,15 +262,6 @@ namespace jni #if defined(WEBRTC_MAC) options.set_allow_iosurface(true); - // Screens are captured with ScreenCaptureKit on macOS 14 and newer. - // The CGDisplayStream capturer otherwise used gets new pixels only - // when its stream calls back, and when that stops it keeps sending the - // first frame with a live cursor on top (issue #344). Older macOS - // falls back to it. Only screens: window capture ignores this, and the - // source list stays with the CGDisplayStream capturer in - // DesktopCapturer, since ScreenCaptureKit lists no screens without its - // system picker. Both name a screen by its CGDirectDisplayID. - options.set_allow_sck_capturer(true); #endif #if defined(WEBRTC_WIN) options.set_allow_directx_capturer(true); @@ -281,6 +277,22 @@ namespace jni std::unique_ptr inner; +#if defined(WEBRTC_MAC) + if (sourceIsWindow) { + // Streams the window with ScreenCaptureKit on macOS 14 and newer, + // instead of a CGWindowListCreateImage snapshot per frame. + inner = CreateScreenCaptureKitWindowCapturer(options); + } + else { + // A screen is captured with ScreenCaptureKit on macOS 14 and newer. + // The CGDisplayStream capturer otherwise used gets new pixels only + // when its stream calls back, and when that stops it keeps sending + // the first frame with a live cursor on top (issue #344). It still + // captures the whole desktop, which ScreenCaptureKit cannot. + inner = CreateMacOSScreenCapturer(options); + } +#endif + #if defined(WEBRTC_USE_PIPEWIRE) // Wayland: request "any screen content" so the xdg-desktop-portal picker // offers both monitors and application windows in a single dialog (the diff --git a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp index 0492624b3..edd91dcc2 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCapturer.cpp @@ -22,6 +22,11 @@ #include "modules/desktop_capture/desktop_and_cursor_composer.h" #include "rtc_base/logging.h" +#if defined(WEBRTC_MAC) +#include "media/video/desktop/macos/MacOSScreenCapturer.h" +#include "media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h" +#endif + namespace jni { DesktopCapturer::DesktopCapturer(bool screenCapturer) : @@ -47,9 +52,21 @@ namespace jni options.set_prefer_cursor_embedded(true); #endif - std::unique_ptr inner = screenCapturer - ? webrtc::DesktopCapturer::CreateScreenCapturer(options) - : webrtc::DesktopCapturer::CreateWindowCapturer(options); + std::unique_ptr inner; + +#if defined(WEBRTC_MAC) + // Both list their sources as WebRTC's capturers do, and capture with + // ScreenCaptureKit on macOS 14 and newer. + inner = screenCapturer + ? CreateMacOSScreenCapturer(options) + : CreateScreenCaptureKitWindowCapturer(options); +#endif + + if (!inner) { + inner = screenCapturer + ? webrtc::DesktopCapturer::CreateScreenCapturer(options) + : webrtc::DesktopCapturer::CreateWindowCapturer(options); + } if (!inner) { RTC_LOG(LS_ERROR) << (screenCapturer diff --git a/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/MacOSScreenCapturer.cpp b/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/MacOSScreenCapturer.cpp new file mode 100644 index 000000000..9313de7e3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/MacOSScreenCapturer.cpp @@ -0,0 +1,141 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/desktop/macos/MacOSScreenCapturer.h" + +#include "modules/desktop_capture/mac/screen_capturer_sck.h" +#include "rtc_base/logging.h" + +namespace jni +{ + namespace + { + class MacOSScreenCapturer final : public webrtc::DesktopCapturer + { + public: + MacOSScreenCapturer(const webrtc::DesktopCaptureOptions & options, + std::unique_ptr displayStream) : + options(options), + displayStream(std::move(displayStream)), + active(nullptr), + screen(webrtc::kFullDesktopScreenId), + maxFrameRate(0) + { + } + + void Start(Callback * callback) override + { + if (screen != webrtc::kFullDesktopScreenId && webrtc::ScreenCapturerSckAvailable()) { + webrtc::DesktopCaptureOptions sckOptions = options; + sckOptions.set_allow_sck_capturer(true); + // ScreenCaptureKit's frames do not say where the screen + // lies on the desktop, which DesktopAndCursorComposer + // needs to place its cursor on any but the first screen. + sckOptions.set_prefer_cursor_embedded(true); + + screenCaptureKit = webrtc::DesktopCapturer::CreateScreenCapturer(sckOptions); + } + + if (screenCaptureKit && screenCaptureKit->SelectSource(screen)) { + active = screenCaptureKit.get(); + } + else { + active = displayStream.get(); + } + + if (maxFrameRate > 0) { + active->SetMaxFrameRate(maxFrameRate); + } + + active->Start(callback); + } + + void SetMaxFrameRate(uint32_t maxFrameRate) override + { + this->maxFrameRate = maxFrameRate; + + if (active) { + active->SetMaxFrameRate(maxFrameRate); + } + } + + void CaptureFrame() override + { + if (active) { + active->CaptureFrame(); + } + } + + bool GetSourceList(SourceList * sources) override + { + return displayStream->GetSourceList(sources); + } + + bool SelectSource(SourceId id) override + { + if (active) { + // Selected after Start(): stays with the backend chosen + // then. + return active->SelectSource(id); + } + + if (!displayStream->SelectSource(id)) { + return false; + } + + screen = id; + + return true; + } + + bool FocusOnSelectedSource() override + { + return active ? active->FocusOnSelectedSource() : false; + } + + bool IsOccluded(const webrtc::DesktopVector & pos) override + { + return active ? active->IsOccluded(pos) : false; + } + + private: + const webrtc::DesktopCaptureOptions options; + + // Lists the screens, and captures where ScreenCaptureKit does not. + const std::unique_ptr displayStream; + std::unique_ptr screenCaptureKit; + webrtc::DesktopCapturer * active; + + SourceId screen; + uint32_t maxFrameRate; + }; + } + + std::unique_ptr CreateMacOSScreenCapturer(const webrtc::DesktopCaptureOptions & options) + { + webrtc::DesktopCaptureOptions displayStreamOptions = options; + displayStreamOptions.set_allow_sck_capturer(false); + + std::unique_ptr displayStream = + webrtc::DesktopCapturer::CreateScreenCapturer(displayStreamOptions); + + if (!displayStream) { + return nullptr; + } + + return std::make_unique(options, std::move(displayStream)); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.cpp b/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.cpp new file mode 100644 index 000000000..24ab116cd --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/desktop/macos/ScreenCaptureKitWindowCapturer.cpp @@ -0,0 +1,715 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/desktop/macos/ScreenCaptureKitWindowCapturer.h" + +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/mac/desktop_frame_iosurface.h" +#include "modules/desktop_capture/mac/window_list_utils.h" +#include "modules/desktop_capture/shared_desktop_frame.h" +#include "rtc_base/logging.h" + +#import +#import +#import + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace jni +{ + class ScreenCaptureKitWindowCapturer; +} + +// Receives the frames of an SCStream. ScreenCaptureKit may still call it +// while the stream stops, so the capturer detaches itself as its owner, under +// the same lock the callbacks take, before it goes away. +API_AVAILABLE(macos(14.0)) +@interface WindowCaptureReceiver : NSObject { +@public + jni::ScreenCaptureKitWindowCapturer * owner; +} +@end + +namespace jni +{ + namespace + { + // How long a ScreenCaptureKit request may take before the capturer + // falls back to WebRTC's window capturer. + constexpr std::chrono::seconds kRequestTimeout(10); + + std::string Describe(NSError * error) + { + if (!error) { + return "no error given"; + } + + const char * reason = error.localizedDescription.UTF8String; + + return "error " + std::to_string(error.code) + (reason ? std::string(": ") + reason : std::string()); + } + + // Returns the process that owns the window, or 0 once it is gone. + pid_t GetWindowOwner(CGWindowID window) + { + // The array holds the ID itself, as a pointer-sized value. + const void * id = reinterpret_cast(static_cast(window)); + CFArrayRef ids = CFArrayCreate(nullptr, &id, 1, nullptr); + CFArrayRef windows = ids ? CGWindowListCreateDescriptionFromArray(ids) : nullptr; + pid_t owner = 0; + + if (windows && CFArrayGetCount(windows) > 0) { + owner = webrtc::GetWindowOwnerPid(static_cast(CFArrayGetValueAtIndex(windows, 0))); + } + + if (windows) { + CFRelease(windows); + } + if (ids) { + CFRelease(ids); + } + + return owner; + } + + // The outcome of an asynchronous ScreenCaptureKit request. Its handler + // holds it, so it outlives a caller that gave up waiting. + struct API_AVAILABLE(macos(14.0)) Request + { + std::mutex mutex; + std::condition_variable condition; + bool done = false; + std::atomic abandoned { false }; + NSError * error = nil; + SCShareableContent * content = nil; + + ~Request() + { + [error release]; + [content release]; + } + + void Finish(NSError * e, SCShareableContent * c = nil) noexcept + { + try { + std::lock_guard lock(mutex); + + error = [e retain]; + content = [c retain]; + done = true; + + condition.notify_all(); + } + catch (...) { + // The waiting caller times out; never unwind into the OS. + } + } + + // Returns false if the request did not finish in time. + bool Wait() + { + std::unique_lock lock(mutex); + + if (!condition.wait_for(lock, kRequestTimeout, [this]() { return done; })) { + abandoned = true; + return false; + } + + return true; + } + }; + } + + class API_AVAILABLE(macos(14.0)) ScreenCaptureKitWindowCapturer final : public webrtc::DesktopCapturer + { + public: + explicit ScreenCaptureKitWindowCapturer(const webrtc::DesktopCaptureOptions & options); + ~ScreenCaptureKitWindowCapturer() override; + + // DesktopCapturer implementation. All run on the caller's thread. + void Start(Callback * callback) override; + void SetMaxFrameRate(uint32_t maxFrameRate) override; + void CaptureFrame() override; + bool GetSourceList(SourceList * sources) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + bool IsOccluded(const webrtc::DesktopVector & pos) override; + + // Called by the receiver, on the stream's queue. + void OnSample(CMSampleBufferRef sample); + void OnStopped(NSError * error); + + private: + // Streams the selected window, or has the fallback capture it. + void StartCapture(); + // Starts streaming the selected window. On failure, returns false + // with the reason set and nothing left running. + bool StartStream(std::string * reason); + void StopStream(); + void StartFallback(); + // Returns a retained configuration for an output of the given size. + SCStreamConfiguration * CreateConfiguration(const webrtc::DesktopSize & size) const; + + const webrtc::DesktopCaptureOptions options; + + Callback * callback; + CGWindowID window; + uint32_t maxFrameRate; + + SCStream * stream; + WindowCaptureReceiver * receiver; + dispatch_queue_t queue; + // The output size the stream has been asked for, in pixels. + webrtc::DesktopSize streamSize; + + // WebRTC's window capturer, once ScreenCaptureKit could not start. + std::unique_ptr fallback; + + // Guards the members below, which the stream's queue writes. + std::mutex mutex; + std::unique_ptr latestFrame; + bool frameIsNew; + bool streamStopped; + // The output size the window's content needs to fit unscaled. + webrtc::DesktopSize neededSize; + }; + + ScreenCaptureKitWindowCapturer::ScreenCaptureKitWindowCapturer(const webrtc::DesktopCaptureOptions & options) : + options(options), + callback(nullptr), + window(kCGNullWindowID), + maxFrameRate(0), + stream(nil), + receiver(nil), + queue(nullptr), + frameIsNew(false), + streamStopped(false) + { + } + + ScreenCaptureKitWindowCapturer::~ScreenCaptureKitWindowCapturer() + { + StopStream(); + } + + void ScreenCaptureKitWindowCapturer::Start(Callback * callback) + { + this->callback = callback; + + StartCapture(); + } + + void ScreenCaptureKitWindowCapturer::SetMaxFrameRate(uint32_t maxFrameRate) + { + this->maxFrameRate = maxFrameRate; + + if (fallback) { + fallback->SetMaxFrameRate(maxFrameRate); + } + else if (stream) { + SCStreamConfiguration * config = CreateConfiguration(streamSize); + + [stream updateConfiguration:config completionHandler:nil]; + [config release]; + } + } + + void ScreenCaptureKitWindowCapturer::CaptureFrame() + { + if (fallback) { + fallback->CaptureFrame(); + return; + } + + if (!stream || GetWindowOwner(window) == 0) { + // Not capturing, or the window was closed. For a closed window, + // ScreenCaptureKit also stops the stream. + callback->OnCaptureResult(Result::ERROR_PERMANENT, nullptr); + return; + } + + std::unique_ptr frame; + webrtc::DesktopSize size; + bool stopped; + + { + std::lock_guard lock(mutex); + + stopped = streamStopped; + size = neededSize; + + if (latestFrame) { + frame = latestFrame->Share(); + + if (!frameIsNew) { + frame->mutable_updated_region()->Clear(); + } + + frameIsNew = false; + } + } + + if (stopped) { + callback->OnCaptureResult(Result::ERROR_PERMANENT, nullptr); + return; + } + + if (!size.is_empty() && !size.equals(streamSize)) { + // The window was resized. Until the stream follows, its frames + // show the window scaled down or with a margin, which the + // receiver crops. + RTC_LOG(LS_INFO) << "ScreenCaptureKit window capture: output " << streamSize.width() << "x" + << streamSize.height() << " -> " << size.width() << "x" << size.height(); + + SCStreamConfiguration * config = CreateConfiguration(size); + + [stream updateConfiguration:config completionHandler:nil]; + [config release]; + + streamSize = size; + } + + if (frame) { + callback->OnCaptureResult(Result::SUCCESS, std::move(frame)); + } + else { + callback->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr); + } + } + + bool ScreenCaptureKitWindowCapturer::GetSourceList(SourceList * sources) + { + // The same list as WebRTC's window capturer, so that its IDs select + // the same windows. + return webrtc::GetWindowList(sources, true, true); + } + + bool ScreenCaptureKitWindowCapturer::SelectSource(SourceId id) + { + if (GetWindowOwner(static_cast(id)) == 0) { + return false; + } + + window = static_cast(id); + + if (fallback) { + return fallback->SelectSource(id); + } + + if (stream) { + StopStream(); + StartCapture(); + } + + return true; + } + + bool ScreenCaptureKitWindowCapturer::FocusOnSelectedSource() + { + if (fallback) { + return fallback->FocusOnSelectedSource(); + } + + const pid_t pid = GetWindowOwner(window); + + if (pid == 0) { + return false; + } + + @autoreleasepool { + // Like WebRTC's window capturer, this brings the whole application + // to the front, not only the window. + NSRunningApplication * app = [NSRunningApplication runningApplicationWithProcessIdentifier:pid]; + + return app && [app activateWithOptions:0]; + } + } + + bool ScreenCaptureKitWindowCapturer::IsOccluded(const webrtc::DesktopVector & pos) + { + // ScreenCaptureKit draws the cursor itself, only where the window is + // visible, so only the fallback needs the composer to know. + return fallback ? fallback->IsOccluded(pos) : false; + } + + void ScreenCaptureKitWindowCapturer::OnSample(CMSampleBufferRef sample) + { + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false); + + if (!attachments || CFArrayGetCount(attachments) < 1) { + return; + } + + NSDictionary * info = (__bridge NSDictionary *) CFArrayGetValueAtIndex(attachments, 0); + NSNumber * status = info[SCStreamFrameInfoStatus]; + + if (!status) { + return; + } + + const SCFrameStatus frameStatus = static_cast(status.integerValue); + + if (frameStatus != SCFrameStatusComplete && frameStatus != SCFrameStatusStarted) { + // Idle when nothing changed; blank or suspended while the window + // cannot be seen, as when minimized. The last frame stays. + return; + } + + CVPixelBufferRef pixels = CMSampleBufferGetImageBuffer(sample); + IOSurfaceRef surface = pixels ? CVPixelBufferGetIOSurface(pixels) : nullptr; + + if (!surface) { + return; + } + + // Pixels per point of the display the window is on. + NSNumber * scaleFactor = info[SCStreamFrameInfoScaleFactor]; + // How far the window had to be scaled down to fit the output. + NSNumber * contentScale = info[SCStreamFrameInfoContentScale]; + + const double scale = scaleFactor && scaleFactor.doubleValue > 0 ? scaleFactor.doubleValue : 1; + const double shrink = contentScale && contentScale.doubleValue > 0 ? contentScale.doubleValue : 1; + + const int surfaceWidth = static_cast(IOSurfaceGetWidth(surface)); + const int surfaceHeight = static_cast(IOSurfaceGetHeight(surface)); + + // Where the window's content lies in the surface, in points. + CGRect content = CGRectMake(0, 0, surfaceWidth / scale, surfaceHeight / scale); + CFDictionaryRef contentRect = (__bridge CFDictionaryRef) info[SCStreamFrameInfoContentRect]; + + if (contentRect) { + CGRectMakeWithDictionaryRepresentation(contentRect, &content); + } + + // The same in pixels, kept inside the surface: DesktopFrameIOSurface + // stops the process on a crop that leaves it. + const int x = std::clamp(static_cast(std::lround(content.origin.x * scale)), 0, surfaceWidth); + const int y = std::clamp(static_cast(std::lround(content.origin.y * scale)), 0, surfaceHeight); + const int width = std::clamp(static_cast(std::lround(content.size.width * scale)), 0, surfaceWidth - x); + const int height = std::clamp(static_cast(std::lround(content.size.height * scale)), 0, surfaceHeight - y); + + if (width <= 0 || height <= 0) { + return; + } + + std::unique_ptr frame = webrtc::DesktopFrameIOSurface::Wrap( + webrtc::ScopedCFTypeRef(surface, webrtc::RetainPolicy::RETAIN), + CGRectMake(x, y, width, height)); + + if (!frame) { + return; + } + + const int dpi = static_cast(std::lround(webrtc::kStandardDPI * scale)); + + frame->set_dpi(webrtc::DesktopVector(dpi, dpi)); + // Keeps DesktopAndCursorComposer from drawing a second cursor. + frame->set_may_contain_cursor(true); + frame->mutable_updated_region()->SetRect(webrtc::DesktopRect::MakeSize(frame->size())); + + // The output the whole window needs, unscaled: its size on screen in + // pixels. The output grows for a window that was scaled down to fit + // it, and shrinks for one that left a margin. + webrtc::DesktopSize needed( + static_cast(std::lround(width / shrink)), + static_cast(std::lround(height / shrink))); + + CGRect screen = CGRectNull; + CFDictionaryRef screenRect = (__bridge CFDictionaryRef) info[SCStreamFrameInfoScreenRect]; + + if (screenRect && CGRectMakeWithDictionaryRepresentation(screenRect, &screen) && + screen.size.width > 0 && screen.size.height > 0) { + // Exact, where the content's size was rounded when scaled down. + needed.set( + static_cast(std::lround(screen.size.width * scale)), + static_cast(std::lround(screen.size.height * scale))); + } + + std::lock_guard lock(mutex); + + latestFrame = webrtc::SharedDesktopFrame::Wrap(std::move(frame)); + frameIsNew = true; + neededSize = needed; + } + + void ScreenCaptureKitWindowCapturer::OnStopped(NSError * error) + { + RTC_LOG(LS_WARNING) << "ScreenCaptureKit window capture stopped: " << Describe(error); + + std::lock_guard lock(mutex); + + streamStopped = true; + } + + void ScreenCaptureKitWindowCapturer::StartCapture() + { + if (GetWindowOwner(window) == getpid()) { + // A stream keeps a window of its own process alive after the + // window is closed, so the capture would never end. This process + // may capture its own windows without screen recording permission. + RTC_LOG(LS_INFO) << "Window " << window << " belongs to this process; capturing it with " + "CGWindowListCreateImage"; + + StartFallback(); + return; + } + + std::string reason; + + if (!StartStream(&reason)) { + RTC_LOG(LS_WARNING) << "ScreenCaptureKit cannot capture window " << window << " (" << reason + << "); capturing it with CGWindowListCreateImage"; + + StartFallback(); + } + } + + bool ScreenCaptureKitWindowCapturer::StartStream(std::string * reason) + { + @autoreleasepool { + auto shareable = std::make_shared(); + + // Off-screen windows too, such as minimized ones or those on + // another Space. + [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:NO + completionHandler:^(SCShareableContent * content, NSError * error) { + shareable->Finish(error, content); + }]; + + if (!shareable->Wait()) { + *reason = "listing shareable content timed out"; + return false; + } + if (!shareable->content) { + *reason = "no shareable content, " + Describe(shareable->error); + return false; + } + + SCWindow * target = nil; + + for (SCWindow * candidate in shareable->content.windows) { + if (candidate.windowID == window) { + target = candidate; + break; + } + } + + if (!target) { + *reason = "the window is not shareable"; + return false; + } + + SCContentFilter * filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:target]; + const double scale = filter.pointPixelScale > 0 ? filter.pointPixelScale : 1; + + streamSize = webrtc::DesktopSize( + static_cast(std::lround(filter.contentRect.size.width * scale)), + static_cast(std::lround(filter.contentRect.size.height * scale))); + + if (streamSize.is_empty()) { + [filter release]; + + *reason = "the window has no size"; + return false; + } + + SCStreamConfiguration * config = CreateConfiguration(streamSize); + + receiver = [[WindowCaptureReceiver alloc] init]; + receiver->owner = this; + queue = dispatch_queue_create("webrtc-window-capture", DISPATCH_QUEUE_SERIAL); + stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:receiver]; + + [filter release]; + [config release]; + + NSError * error = nil; + + if (![stream addStreamOutput:receiver type:SCStreamOutputTypeScreen sampleHandlerQueue:queue error:&error]) { + *reason = "cannot add the stream output, " + Describe(error); + + StopStream(); + return false; + } + + auto started = std::make_shared(); + SCStream * starting = stream; + + [stream startCaptureWithCompletionHandler:^(NSError * startError) { + // A start that finishes after the caller gave up must not + // leave the window captured unseen. + if (started->abandoned.load() && !startError) { + [starting stopCaptureWithCompletionHandler:nil]; + } + + started->Finish(startError); + }]; + + if (!started->Wait()) { + *reason = "starting the stream timed out"; + + StopStream(); + return false; + } + if (started->error) { + *reason = "cannot start the stream, " + Describe(started->error); + + StopStream(); + return false; + } + + RTC_LOG(LS_INFO) << "ScreenCaptureKit window capture: window " << window << ", output " + << streamSize.width() << "x" << streamSize.height(); + + return true; + } + } + + void ScreenCaptureKitWindowCapturer::StopStream() + { + @autoreleasepool { + if (receiver) { + @synchronized(receiver) { + receiver->owner = nullptr; + } + } + + if (stream) { + SCStream * stopping = stream; + WindowCaptureReceiver * stoppingReceiver = receiver; + dispatch_queue_t stoppingQueue = queue; + + // The handler keeps the stream, its receiver and its queue + // until ScreenCaptureKit has stopped using them. + [stream stopCaptureWithCompletionHandler:^(NSError * error) { + [stopping self]; + [stoppingReceiver self]; + (void) stoppingQueue; + }]; + } + + [stream release]; + [receiver release]; + + if (queue) { + dispatch_release(queue); + } + + stream = nil; + receiver = nil; + queue = nullptr; + + std::lock_guard lock(mutex); + + latestFrame.reset(); + frameIsNew = false; + streamStopped = false; + neededSize = webrtc::DesktopSize(); + } + } + + void ScreenCaptureKitWindowCapturer::StartFallback() + { + fallback = webrtc::DesktopCapturer::CreateWindowCapturer(options); + + if (!fallback) { + return; + } + + fallback->SelectSource(window); + + if (maxFrameRate > 0) { + fallback->SetMaxFrameRate(maxFrameRate); + } + + fallback->Start(callback); + } + + SCStreamConfiguration * ScreenCaptureKitWindowCapturer::CreateConfiguration(const webrtc::DesktopSize & size) const + { + SCStreamConfiguration * config = [[SCStreamConfiguration alloc] init]; + + config.width = size.width(); + config.height = size.height(); + config.pixelFormat = kCVPixelFormatType_32BGRA; + config.colorSpaceName = kCGColorSpaceSRGB; + // A window larger than the output is scaled down, never up; the next + // CaptureFrame() then asks for an output it fits. + config.scalesToFit = NO; + // Without the shadow, as CGWindowListCreateImage with + // kCGWindowImageBoundsIgnoreFraming. + config.ignoreShadowsSingleWindow = YES; + config.showsCursor = YES; + config.minimumFrameInterval = maxFrameRate > 0 + ? CMTimeMake(1, static_cast(maxFrameRate)) + : kCMTimeZero; + + return config; + } + + std::unique_ptr CreateScreenCaptureKitWindowCapturer(const webrtc::DesktopCaptureOptions & options) + { + if (@available(macOS 14.0, *)) { + return std::make_unique(options); + } + + return nullptr; + } +} + +@implementation WindowCaptureReceiver + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sample ofType:(SCStreamOutputType)type +{ + if (type != SCStreamOutputTypeScreen || !CMSampleBufferIsValid(sample)) { + return; + } + + @synchronized(self) { + try { + if (owner) { + owner->OnSample(sample); + } + } + catch (...) { + // Drops the frame; never unwind into ScreenCaptureKit. + } + } +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error +{ + @synchronized(self) { + try { + if (owner) { + owner->OnStopped(error); + } + } + catch (...) { + } + } +} + +@end diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java index eab0f05c9..583a9a75d 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java @@ -27,21 +27,26 @@ import dev.onvoid.webrtc.media.MediaStreamTrackState; import dev.onvoid.webrtc.media.video.desktop.DesktopSource; import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer; +import dev.onvoid.webrtc.media.video.desktop.TestWindowProcess; import java.awt.Color; import java.awt.EventQueue; import java.awt.Frame; +import java.awt.GraphicsConfiguration; +import java.awt.GraphicsDevice; import java.awt.GraphicsEnvironment; import java.awt.Rectangle; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; @@ -203,6 +208,178 @@ void screenFramesFollowScreen() throws Exception { + String.format(Locale.ROOT, "%.1f to %.1f", min, max)); } + @Test + void windowFramesFollowWindow() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + double scale = screenScale(); + + // A window of another process, which ScreenCaptureKit captures on + // macOS 14 and newer. This process's own windows are captured with + // CGWindowListCreateImage. + try (TestWindowProcess window = TestWindowProcess.open(400, 300)) { + DesktopSource source = window.find(); + + // Without screen recording permission, another process's windows + // are listed without their titles. + assumeTrue(source != null, "The window is not listed under its title"); + + try (WindowCapture capture = new WindowCapture(source)) { + int[] first = capture.await(frame -> true, 5000); + + assumeTrue(first != null, "The window delivered no frames"); + // Black but for the title bar, unless the content is hidden + // for want of permission. + assumeTrue(first[2] < 80, "The window's content is not visible"); + + assertSize(400, 300, scale, first); + + window.setColor(true); + + int[] white = capture.await(frame -> frame[2] > 150, 3000); + + assertTrue(white[2] > 150, "Frames did not follow the window turning white: luma " + white[2]); + + window.setSize(800, 500); + assertSize(800, 500, scale, capture.await(frame -> hasSize(800, 500, scale, frame), 3000)); + + window.setSize(300, 250); + assertSize(300, 250, scale, capture.await(frame -> hasSize(300, 250, scale, frame), 3000)); + } + } + } + + @Test + void closedWindowStopsCapture() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + try (TestWindowProcess window = TestWindowProcess.open(400, 300)) { + DesktopSource source = window.find(); + + assumeTrue(source != null, "The window is not listed under its title"); + + try (WindowCapture capture = new WindowCapture(source)) { + assumeTrue(capture.await(frame -> true, 5000) != null, "The window delivered no frames"); + + window.closeWindow(); + + // Capture stops within a frame or two of the window closing. + Thread.sleep(1000); + + int frames = capture.frames(); + + Thread.sleep(1000); + + assertEquals(frames, capture.frames(), "Frames kept coming after the window closed"); + } + } + } + + @Test + void ownWindowCapturedUntilClosed() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + double scale = screenScale(); + + // This process captures its own window with CGWindowListCreateImage, + // which needs no screen recording permission, and closing the window + // must end the capture. Its content is not checked: a window of a + // process that is not in front may not be redrawn in time. + String title = "webrtc-java " + UUID.randomUUID(); + Frame window = new Frame(title); + + EventQueue.invokeAndWait(() -> { + window.setBackground(Color.BLACK); + window.setBounds(100, 100, 400, 300); + window.setVisible(true); + }); + + try { + DesktopSource source = TestWindowProcess.findWindow(title); + + assumeTrue(source != null, "The window is not listed under its title"); + + try (WindowCapture capture = new WindowCapture(source)) { + int[] first = capture.await(frame -> true, 5000); + + assumeTrue(first != null, "The window delivered no frames"); + + assertSize(400, 300, scale, first); + + EventQueue.invokeAndWait(window::dispose); + + Thread.sleep(1000); + + int frames = capture.frames(); + + Thread.sleep(1000); + + assertEquals(frames, capture.frames(), "Frames kept coming after the window closed"); + } + } + finally { + EventQueue.invokeAndWait(window::dispose); + } + } + + @Test + void fullDesktopCoversAllScreens() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS screen capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + // The desktop, as WebRTC's CGDisplayStream capturer composes it: the + // bounds of all screens, in pixels. ScreenCaptureKit captures single + // screens only, and would deliver just the first. + Rectangle desktop = new Rectangle(); + double scale = 0; + + for (GraphicsDevice device : GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices()) { + GraphicsConfiguration config = device.getDefaultConfiguration(); + double deviceScale = config.getDefaultTransform().getScaleX(); + + assumeTrue(scale == 0 || scale == deviceScale, "The screens differ in scale"); + + scale = deviceScale; + desktop = desktop.isEmpty() ? config.getBounds() : desktop.union(config.getBounds()); + } + + VideoDesktopSource source = new VideoDesktopSource(); + source.setSourceId(-1, false); + source.setFrameRate(10); + + VideoTrack track = factory.createVideoTrack("desktop", source); + + CompletableFuture size = new CompletableFuture<>(); + + VideoTrackSink sink = frame -> { + size.complete(new int[] { frame.buffer.getWidth(), frame.buffer.getHeight() }); + frame.release(); + }; + + int[] first; + + try { + track.addSink(sink); + source.start(); + + first = size.completeOnTimeout(null, 10, TimeUnit.SECONDS).join(); + } + finally { + source.stop(); + track.removeSink(sink); + track.dispose(); + source.dispose(); + } + + assumeTrue(first != null, "The desktop delivered no frames"); + + assertEquals(Math.round(desktop.width * scale) + "x" + Math.round(desktop.height * scale), + first[0] + "x" + first[1]); + } + @Test void retainedFrameKeepsPixels() throws Exception { assumeFalse(isWaylandSession(), "Starting a capture asks the user what to share"); @@ -321,6 +498,103 @@ void missingBackendEndsTrack() { } } + /** + * Captures one window, keeping the width, height and mean luma of the + * latest frame. + */ + private final class WindowCapture implements AutoCloseable { + + private final VideoDesktopSource source = new VideoDesktopSource(); + + private final VideoTrack track; + + private final VideoTrackSink sink; + + private final AtomicInteger frames = new AtomicInteger(); + + private final AtomicReference latest = new AtomicReference<>(); + + + WindowCapture(DesktopSource window) { + source.setSourceId(window.id, true); + source.setFrameRate(10); + + track = factory.createVideoTrack("window", source); + sink = frame -> { + NativeI420Buffer buffer = (NativeI420Buffer) frame.buffer; + + latest.set(new int[] { buffer.getWidth(), buffer.getHeight(), (int) sampledLuma(buffer) }); + frame.release(); + frames.incrementAndGet(); + }; + + track.addSink(sink); + source.start(); + } + + /** + * Waits for a frame that meets the condition. Returns it, or else the + * latest frame, which is null if none came. + */ + int[] await(Predicate condition, long millis) throws InterruptedException { + long deadline = System.currentTimeMillis() + millis; + int[] frame = latest.get(); + + while ((frame == null || !condition.test(frame)) && System.currentTimeMillis() < deadline) { + Thread.sleep(50); + + frame = latest.get(); + } + + return frame; + } + + int frames() { + return frames.get(); + } + + @Override + public void close() { + source.stop(); + track.removeSink(sink); + track.dispose(); + source.dispose(); + } + } + + private static double screenScale() { + return GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice() + .getDefaultConfiguration().getDefaultTransform().getScaleX(); + } + + private static boolean hasSize(int width, int height, double scale, int[] frame) { + return Math.abs(frame[0] - Math.round(width * scale)) <= 2 + && Math.abs(frame[1] - Math.round(height * scale)) <= 2; + } + + private static void assertSize(int width, int height, double scale, int[] frame) { + assertTrue(frame != null && hasSize(width, height, scale, frame), + "Expected a frame of a " + width + "x" + height + " window at scale " + scale + ", got " + + (frame == null ? "none" : frame[0] + "x" + frame[1])); + } + + /** The mean luma of every fourth pixel in every fourth row. */ + private static double sampledLuma(NativeI420Buffer buffer) { + ByteBuffer data = buffer.getDataY(); + int stride = buffer.getStrideY(); + long sum = 0; + long count = 0; + + for (int y = 0; y < buffer.getHeight(); y += 4) { + for (int x = 0; x < buffer.getWidth(); x += 4) { + sum += data.get(y * stride + x) & 0xFF; + count++; + } + } + + return (double) sum / count; + } + private static ByteBuffer copyOf(ByteBuffer buffer) { ByteBuffer copy = ByteBuffer.allocate(buffer.remaining()); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java index 5fc4eecd1..683007845 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/DesktopCapturerTests.java @@ -18,13 +18,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import dev.onvoid.webrtc.TestBase; - +import dev.onvoid.webrtc.media.video.I420Buffer; + +import java.awt.Color; +import java.awt.EventQueue; +import java.awt.Frame; +import java.awt.GraphicsEnvironment; +import java.awt.Rectangle; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; @@ -80,6 +89,133 @@ void missingBackendReportsPermanentError() { } } + @Test + void windowCapturerFollowsWindowUntilClosed() throws Exception { + assumeTrue(System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac"), + "covers the macOS window capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + double scale = GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice() + .getDefaultConfiguration().getDefaultTransform().getScaleX(); + + // A window of another process, which ScreenCaptureKit captures on + // macOS 14 and newer. + try (TestWindowProcess window = TestWindowProcess.open(400, 300)) { + DesktopSource source = window.find(); + + assumeTrue(source != null, "The window is not listed under its title"); + + WindowCapturer capturer = new WindowCapturer(); + List results = new ArrayList<>(); + + try { + capturer.selectSource(source); + capturer.start((result, frame) -> { + if (frame != null) { + results.add(frame.buffer.getWidth() + "x" + frame.buffer.getHeight()); + frame.release(); + } + else { + results.add(result.name()); + } + }); + + String size = Math.round(400 * scale) + "x" + Math.round(300 * scale); + + for (int i = 0; i < 50 && !results.contains(size); i++) { + capturer.captureFrame(); + Thread.sleep(100); + } + + assumeTrue(results.stream().anyMatch(r -> r.contains("x")), "The window delivered no frames"); + assertTrue(results.contains(size), "No " + size + " frame of the window: " + results); + + window.closeWindow(); + + for (int i = 0; i < 30 && !results.contains("ERROR_PERMANENT"); i++) { + capturer.captureFrame(); + Thread.sleep(100); + } + + assertEquals("ERROR_PERMANENT", results.get(results.size() - 1), + "The closed window was still captured: " + results); + } + finally { + capturer.dispose(); + } + } + } + + @Test + void screenCapturerFollowsScreen() throws Exception { + assumeTrue(System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac"), + "covers the macOS screen capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + // A window of our own on the main screen, switched between black and + // white. A capture that froze on its first frame keeps showing one of + // the two (issue #344). + Frame window = new Frame(); + Rectangle bounds = GraphicsEnvironment.getLocalGraphicsEnvironment() + .getDefaultScreenDevice().getDefaultConfiguration().getBounds(); + + EventQueue.invokeAndWait(() -> { + window.setUndecorated(true); + window.setAlwaysOnTop(true); + window.setBackground(Color.BLACK); + window.setBounds(bounds.x + bounds.width / 10, bounds.y + bounds.height / 10, + bounds.width * 8 / 10, bounds.height * 8 / 10); + window.setVisible(true); + }); + + ScreenCapturer capturer = new ScreenCapturer(); + List luma = new ArrayList<>(); + + try { + List screens = capturer.getDesktopSources(); + + assumeFalse(screens.isEmpty(), "No screen to capture"); + + // macOS lists the main screen first. + capturer.selectSource(screens.get(0)); + capturer.start((result, frame) -> { + if (frame != null) { + luma.add(sampledLuma(frame.buffer.toI420())); + frame.release(); + } + }); + + // Calls back on this thread, so the list needs no lock. + for (int i = 0; i < 40; i++) { + if (i % 5 == 0) { + Color color = i / 5 % 2 == 0 ? Color.BLACK : Color.WHITE; + + EventQueue.invokeAndWait(() -> { + window.setBackground(color); + window.repaint(); + }); + } + + capturer.captureFrame(); + Thread.sleep(100); + } + } + finally { + capturer.dispose(); + + EventQueue.invokeAndWait(window::dispose); + } + + assumeFalse(luma.isEmpty(), "The screen delivered no frames"); + + double min = luma.stream().mapToDouble(Double::doubleValue).min().getAsDouble(); + double max = luma.stream().mapToDouble(Double::doubleValue).max().getAsDouble(); + + // The window covers about two thirds of the screen. + assertTrue(max - min > 50, "Frames did not follow the screen: " + luma.size() + + " frames, mean luma " + String.format(Locale.ROOT, "%.1f to %.1f", min, max)); + } + /** * Starting a capturer in a Wayland session asks the user what to share. */ @@ -87,6 +223,23 @@ private static void assumeNoPortalDialog() { assumeFalse("wayland".equals(System.getenv("XDG_SESSION_TYPE")), "Wayland session"); } + /** The mean luma of every fourth pixel in every fourth row. */ + private static double sampledLuma(I420Buffer buffer) { + ByteBuffer data = buffer.getDataY(); + int stride = buffer.getStrideY(); + long sum = 0; + long count = 0; + + for (int y = 0; y < buffer.getHeight(); y += 4) { + for (int x = 0; x < buffer.getWidth(); x += 4) { + sum += data.get(y * stride + x) & 0xFF; + count++; + } + } + + return (double) sum / count; + } + private static boolean hasNoBackend(DesktopCapturer capturer) { try { capturer.getDesktopSources(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/TestWindowProcess.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/TestWindowProcess.java new file mode 100644 index 000000000..c3fa1a383 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/desktop/TestWindowProcess.java @@ -0,0 +1,172 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.desktop; + +import java.awt.Color; +import java.awt.EventQueue; +import java.awt.Frame; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +/** + * A black window in a process of its own, for capturing a window that this + * process does not own. The test drives it through the child's standard + * input. + */ +public final class TestWindowProcess implements AutoCloseable { + + private final Process process; + + private final PrintWriter commands; + + private final String title; + + + private TestWindowProcess(Process process, String title) { + this.process = process; + this.title = title; + this.commands = new PrintWriter(process.getOutputStream(), true, StandardCharsets.UTF_8); + } + + /** + * Opens a window of the given outer size, in points. + */ + public static TestWindowProcess open(int width, int height) throws Exception { + String title = "webrtc-java " + UUID.randomUUID(); + Path classes = Path.of(TestWindowProcess.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + Path java = Path.of(System.getProperty("java.home"), "bin", "java"); + + Process process = new ProcessBuilder(java.toString(), "-cp", classes.toString(), + Child.class.getName(), title, Integer.toString(width), Integer.toString(height)) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + + return new TestWindowProcess(process, title); + } + + /** + * Returns the window as listed by a WindowCapturer, or null if it does + * not show up within a few seconds. Without screen recording permission, + * the windows of other processes are listed without their titles. + */ + public DesktopSource find() throws InterruptedException { + return findWindow(title); + } + + public void setColor(boolean white) { + commands.println(white ? "white" : "black"); + } + + public void setSize(int width, int height) { + commands.println("size " + width + " " + height); + } + + public void closeWindow() { + commands.println("close"); + } + + @Override + public void close() throws InterruptedException { + commands.println("exit"); + + if (!process.waitFor(5, TimeUnit.SECONDS)) { + process.destroyForcibly(); + } + } + + /** + * Returns the window of the given title as listed by a WindowCapturer, or + * null if it does not show up within a few seconds. + */ + public static DesktopSource findWindow(String title) throws InterruptedException { + WindowCapturer capturer = new WindowCapturer(); + + try { + for (int i = 0; i < 100; i++) { + for (DesktopSource window : capturer.getDesktopSources()) { + if (title.equals(window.title)) { + return window; + } + } + + Thread.sleep(100); + } + } + finally { + capturer.dispose(); + } + + return null; + } + + /** + * The window's process. It needs nothing but AWT on its class path, so it + * refers to no other class of this library. + */ + static final class Child { + + public static void main(String[] args) throws Exception { + Frame window = new Frame(args[0]); + + EventQueue.invokeAndWait(() -> { + window.setBackground(Color.BLACK); + window.setBounds(100, 100, Integer.parseInt(args[1]), Integer.parseInt(args[2])); + window.setVisible(true); + // Not hidden behind other windows, where it may not be redrawn. + window.toFront(); + }); + + BufferedReader input = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + String line; + + while ((line = readLine(input)) != null && !line.equals("exit")) { + String[] command = line.split(" "); + + EventQueue.invokeAndWait(() -> { + if (command[0].equals("white") || command[0].equals("black")) { + window.setBackground(command[0].equals("white") ? Color.WHITE : Color.BLACK); + window.repaint(); + } + else if (command[0].equals("size")) { + window.setSize(Integer.parseInt(command[1]), Integer.parseInt(command[2])); + } + else if (command[0].equals("close")) { + window.dispose(); + } + }); + } + + System.exit(0); + } + + private static String readLine(BufferedReader input) { + try { + return input.readLine(); + } + catch (IOException e) { + return null; + } + } + } +}