From 1d5a7b563827f796ef11446de699bb2ebd63528f Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Wed, 11 Nov 2026 01:03:22 +0100 Subject: [PATCH] feat: hint video tracks as motion, detail or text VideoTrack and LocalVideo take a VideoContentHint, also while sending and without renegotiating. A camera switch keeps it. --- docs/guide/client-api-design.md | 4 +- docs/guide/client-api.md | 20 ++++- docs/guide/media/constraints.md | 13 +++ docs/guide/video/desktop-capture.md | 5 ++ .../webrtc/examples/DesktopVideoExample.java | 5 ++ .../src/main/cpp/include/JNI_VideoTrack.h | 16 ++++ .../src/main/cpp/src/JNI_VideoTrack.cpp | 30 ++++++- webrtc-jni/src/main/cpp/src/WebRTCContext.cpp | 1 + .../onvoid/webrtc/client/LocalTrackImpl.java | 15 +++- .../dev/onvoid/webrtc/client/LocalVideo.java | 25 ++++++ .../onvoid/webrtc/client/LocalVideoImpl.java | 29 +++++++ .../onvoid/webrtc/internal/SinkRegistry.java | 7 +- .../webrtc/media/video/VideoContentHint.java | 60 +++++++++++++ .../onvoid/webrtc/media/video/VideoTrack.java | 41 +++++++++ .../webrtc-java/jni-config.json | 6 ++ .../webrtc-java/reflect-config.json | 7 ++ .../dev/onvoid/webrtc/client/MediaTests.java | 84 +++++++++++++++++++ .../webrtc/media/video/VideoTrackTests.java | 40 +++++++++ 18 files changed, 401 insertions(+), 7 deletions(-) create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoContentHint.java diff --git a/docs/guide/client-api-design.md b/docs/guide/client-api-design.md index 9ca3936a7..75bb5557e 100644 --- a/docs/guide/client-api-design.md +++ b/docs/guide/client-api-design.md @@ -140,7 +140,7 @@ and it hands on to the event executor what the application should hear. | Code | Runs on | May it block? | |---|---|---| -| Calls into the API | the caller's thread, any thread | They don't block, with these exceptions. `createPeer` waits for the connection, so that an invalid configuration fails there. The `LocalMedia` methods that make a track and `LocalTrack.setEnabled` wait briefly for a WebRTC thread, since the factory and the tracks are proxies. `camera()` and `screen()` also open their capture on the caller's thread. The first `devices()`, `screens()` and `windows()` ask the system there. `WebRtc.close()` waits up to `closeTimeout`, and a `join()` the caller writes waits as written. | +| Calls into the API | the caller's thread, any thread | They don't block, with these exceptions. `createPeer` waits for the connection, so that an invalid configuration fails there. The `LocalMedia` methods that make a track and `LocalTrack.setEnabled` and `LocalVideo.setContentHint` wait briefly for a WebRTC thread, since the factory and the tracks are proxies. `camera()` and `screen()` also open their capture on the caller's thread. The first `devices()`, `screens()` and `windows()` ask the system there. `WebRtc.close()` waits up to `closeTimeout`, and a `join()` the caller writes waits as written. | | Event handlers, future callbacks, `Signaling.send` | the peer's event thread, or the configured `eventExecutor` | Yes, but it only delays that peer's events. It must not `join()` a future of this API: a direct `join()`/`get()` throws `BlockingCallException`. | | `LocalTrack.onEnded`, `LocalAudio.onVoiceActivity`, `Devices.onChange`, futures of local tracks and devices | the event thread of the runtime's local media, or the configured `eventExecutor` | Yes, but it delays the other events of local media. The rule against `join()` holds here too. | | Frame handlers (`onFrame`, `onAudio`) | a thread of each subscription, from the frame pool | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped and released. A slow audio handler loses the oldest audio beyond its buffer. | @@ -549,6 +549,8 @@ public interface LocalTrack extends AutoCloseable { public interface LocalVideo extends LocalTrack { Subscription onFrame(Consumer handler); // local preview + VideoContentHint contentHint(); + void setContentHint(VideoContentHint hint); // motion, detail or text, also while sending } public interface LocalCamera extends LocalVideo { diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 40925cbad..6b75fab70 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -290,7 +290,7 @@ Every method of the client API may be called from any thread, including from ins | Code | Runs on | May it block? | |---|---|---| -| Your calls into the API | your thread | They don't block, with these exceptions. `WebRtc.createPeer`, the `LocalMedia` methods that make a track and `LocalTrack.setEnabled` wait briefly for WebRTC. `camera()` and `screen()` also open their capture on your thread, which can take longer, so a UI thread should leave them to another thread. The first `media.devices()`, `screens()` and `windows()` ask the system on your thread. `WebRtc.close()` waits up to the close timeout. | +| Your calls into the API | your thread | They don't block, with these exceptions. `WebRtc.createPeer`, the `LocalMedia` methods that make a track and `LocalTrack.setEnabled` and `LocalVideo.setContentHint` wait briefly for WebRTC. `camera()` and `screen()` also open their capture on your thread, which can take longer, so a UI thread should leave them to another thread. The first `media.devices()`, `screens()` and `windows()` ask the system on your thread. `WebRtc.close()` waits up to the close timeout. | | Peer, track and channel handlers, future callbacks, `Signaling.send` | the peer's event thread | Yes, but that delays only the events of this peer. They must not wait for a future of this API, see below. | | `onFrame` and `onAudio` handlers | a thread of each subscription | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped; a slow audio handler loses the oldest audio beyond its buffer. | | [Encoded transforms](#encoded-transforms) | a thread of each sender and receiver | Better not. A slow transform delays only its own frames, and one that falls several seconds behind has frames dropped. | @@ -483,6 +483,24 @@ Devices devices = media.devices(); LocalVideo screen = media.screen(ScreenOptions.screen(devices.screens().get(0)).frameRate(15).build()); ``` +A content hint tells the senders of a video track what to keep when the bandwidth or the CPU runs short. It can be changed at any time, also while the track is sent, without renegotiating: + +```java +screen.setContentHint(VideoContentHint.TEXT); // source code: keep the resolution +peer.addTrack(screen); + +screen.setContentHint(VideoContentHint.MOTION); // a video starts in the shared window: keep the frame rate +``` + +| Hint | Suits | Gives up first | +|---|---|---| +| `DEFAULT` | Anything | What the source suggests: the frame rate of screens, the resolution of other video | +| `MOTION` | Cameras, videos, animations | Resolution | +| `DETAIL` | Drawings, slides, photos | Frame rate | +| `TEXT` | Source code, terminals, tables | Frame rate; WebRTC treats it as `DETAIL` for now | + +The hint belongs to the track, so it applies on every peer the track is sent on, and a camera keeps it when it switches. Screens keep their resolution by default already, so the hint pays off where the content changes, or for a camera that shows a whiteboard. No hint keeps text readable at any bitrate. + Tracks added without stream ids share one stream per peer, so that a browser plays their audio and video in sync. Pass ids to `addTrack(track, streamIds...)` to group them yourself. One track may be sent on any number of peers. Opening a camera that is already open shares its capture when the options are the same, and fails with a `MediaException` when they differ. diff --git a/docs/guide/media/constraints.md b/docs/guide/media/constraints.md index 532be7a1d..c9fc5c332 100644 --- a/docs/guide/media/constraints.md +++ b/docs/guide/media/constraints.md @@ -142,6 +142,19 @@ sender.setParameters(parameters); If unset, WebRTC keeps the resolution of screen shares and of tracks hinted as detailed or text, and the frame rate of other video. +### Content Hint + +A content hint on the track says what it shows, and so decides the same for every sender of the track that has no `degradationPreference` of its own. It can be changed while the track is sent, without renegotiation, as when a video starts in a shared window: + +```java +videoTrack.setContentHint(VideoContentHint.TEXT); // keep the resolution + +// Later, a video plays in the same window. +videoTrack.setContentHint(VideoContentHint.MOTION); // keep the frame rate +``` + +`MOTION` keeps the frame rate, `DETAIL` and `TEXT` keep the resolution, and `DEFAULT` leaves it to the source, which keeps the resolution of screen shares. `DETAIL` and `TEXT` also put the encoder into its screen content mode, which VP8, VP9, H.264 (OpenH264) and AV1 code differently; WebRTC treats `TEXT` as `DETAIL` for now. The senders have taken the hint once `setContentHint` returns. It cannot be set from within the callback of a video sink, where it throws an `IllegalStateException`. + ## Simulcast With simulcast, a video sender encodes the same video several times, at different resolutions, so that a server can forward to each receiver the one its connection can take. The encodings are given when the transceiver is created, each with an RTP stream ID (`rid`): diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index 9a5d3c483..42dc29314 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -83,8 +83,13 @@ PeerConnectionFactory factory = new PeerConnectionFactory(); // Create a video track with the desktop source VideoTrack videoTrack = factory.createVideoTrack("video0", videoSource); + +// Keep the resolution, so that text stays readable, also when the bandwidth runs short. +videoTrack.setContentHint(VideoContentHint.TEXT); ``` +The [content hint](/guide/media/constraints#content-hint) can be changed at any time, also while the track is sent. + ### Adding the Track to a Peer Connection Add the video track to your peer connection: diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java index 01e580a21..57627848b 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java @@ -31,6 +31,7 @@ import dev.onvoid.webrtc.client.SignalingPair; import dev.onvoid.webrtc.client.VideoStats; import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.media.video.VideoContentHint; import dev.onvoid.webrtc.media.video.desktop.DesktopSource; /** @@ -95,6 +96,10 @@ public static void main(String[] args) throws Exception { // Frames larger than 1920x1080 are scaled down, keeping their aspect ratio. LocalVideo screen = rtc.media().screen(options.frameRate(30).maxSize(1920, 1080).build()); + // Keeps text readable when the bandwidth runs short, lowering the frame rate instead. + // Switch to MOTION while a video plays on the screen, the peers go on sending. + screen.setContentHint(VideoContentHint.TEXT); + System.out.println("Sharing " + describe(source)); // The two ends of a loopback signaling hand each message to the other peer. diff --git a/webrtc-jni/src/main/cpp/include/JNI_VideoTrack.h b/webrtc-jni/src/main/cpp/include/JNI_VideoTrack.h index f9ee877c7..73a3a2703 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_VideoTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_VideoTrack.h @@ -23,6 +23,22 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_removeSinkInternal (JNIEnv *, jobject, jlong); + /* + * Class: dev_onvoid_webrtc_media_video_VideoTrack + * Method: getContentHint + * Signature: ()Ldev/onvoid/webrtc/media/video/VideoContentHint; + */ + JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_getContentHint + (JNIEnv *, jobject); + + /* + * Class: dev_onvoid_webrtc_media_video_VideoTrack + * Method: setContentHintInternal + * Signature: (Ldev/onvoid/webrtc/media/video/VideoContentHint;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_setContentHintInternal + (JNIEnv *, jobject, jobject); + #ifdef __cplusplus } #endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_VideoTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_VideoTrack.cpp index 839309fca..0a22316a7 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_VideoTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_VideoTrack.cpp @@ -18,6 +18,7 @@ #include "JavaBoundary.h" #include "NativeHandle.h" #include "api/VideoTrackSink.h" +#include "JavaEnums.h" #include "JavaNullPointerException.h" #include "JavaUtils.h" @@ -62,4 +63,31 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_removeSinkI delete sink; } }); -} \ No newline at end of file +} + +JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_getContentHint +(JNIEnv * env, jobject caller) +{ + return jni::CallFromJava(env, [&]() -> jobject { + jni::NativeLease track(env, caller); + CHECK_HANDLEV(track, nullptr); + + return jni::JavaEnums::toJava(env, track->content_hint()).release(); + }); +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrack_setContentHintInternal +(JNIEnv * env, jobject caller, jobject jHint) +{ + jni::CallFromJava(env, [&]() { + jni::NativeLease track(env, caller); + CHECK_HANDLE(track); + + auto hint = jni::JavaEnums::toNative(env, jHint); + + // The senders of the track reconfigure before this returns: the + // proxy runs it on the signaling thread, where the track notifies + // them, and each one waits for the worker thread. + track->set_content_hint(hint); + }); +} diff --git a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp index fc9487217..3b3d6304e 100644 --- a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp +++ b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp @@ -89,6 +89,7 @@ namespace jni JavaEnums::add(env, PKG"RTCDtxStatus"); JavaEnums::add(env, PKG_MEDIA"MediaSource$State"); JavaEnums::add(env, PKG_MEDIA"MediaStreamTrackState"); + JavaEnums::add(env, PKG_MEDIA"video/VideoContentHint"); JavaEnums::add(env, PKG"RTCBundlePolicy"); JavaEnums::add(env, PKG"RTCIceConnectionState"); JavaEnums::add(env, PKG"RTCIceGatheringState"); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java index 2dbfeb5a4..bc917ddad 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java @@ -154,8 +154,17 @@ MediaStreamTrack nativeTrack() { } /** - * Puts another native track in place of the current one, keeping whether - * it is enabled. + * Gives a native track that is to replace the current one the settings + * of this track, before any sender sends it. Called under this object's + * lock. + */ + void adopt(MediaStreamTrack next) { + next.setEnabled(enabled); + } + + /** + * Puts another native track in place of the current one, keeping its + * settings, such as whether it is enabled. * * @return the track and release replaced, and the senders that send it now; * null once the track ended. @@ -167,7 +176,7 @@ synchronized Replaced replaceNative(MediaStreamTrack next, Runnable nextRelease) Replaced replaced = new Replaced(track, releaseSource, new ArrayList<>(senders)); - next.setEnabled(enabled); + adopt(next); track = next; releaseSource = nextRelease; diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java index ed9d2458f..f33a66d1c 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java @@ -18,6 +18,7 @@ import java.util.function.Consumer; +import dev.onvoid.webrtc.media.video.VideoContentHint; import dev.onvoid.webrtc.media.video.VideoFrame; /** @@ -37,4 +38,28 @@ public interface LocalVideo extends LocalTrack { */ Subscription onFrame(Consumer handler); + /** + * @return what the track is hinted to show, {@link + * VideoContentHint#DEFAULT} unless set. + */ + VideoContentHint contentHint(); + + /** + * Hints what the track shows, which decides what its senders keep when + * the network or the CPU cannot keep up: the frame rate for {@link + * VideoContentHint#MOTION}, the resolution for {@link + * VideoContentHint#DETAIL} and {@link VideoContentHint#TEXT}. Can be + * changed while the track is sent, without renegotiating, as when a video + * starts playing in a shared window. + *

+ * The hint belongs to the track: it applies on every peer the track is + * sent on, and stays when a camera switches. Every sender has taken the + * hint once this returns; a change may make a sender reconfigure its + * encoder. Waits briefly for a WebRTC thread. Does nothing once the + * track ended. + * + * @param hint what the track shows. + */ + void setContentHint(VideoContentHint hint); + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java index 941d35177..a1f859560 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.function.Consumer; +import dev.onvoid.webrtc.media.MediaStreamTrack; +import dev.onvoid.webrtc.media.video.VideoContentHint; import dev.onvoid.webrtc.media.video.VideoFrame; import dev.onvoid.webrtc.media.video.VideoTrack; @@ -34,6 +36,9 @@ class LocalVideoImpl extends LocalTrackImpl implements LocalVideo { private final List dispatchers = new ArrayList<>(); + /** Written under this, kept for the native track a camera switch makes. */ + private volatile VideoContentHint contentHint = VideoContentHint.DEFAULT; + LocalVideoImpl(LocalMediaImpl media, VideoTrack track, Runnable releaseSource) { super(media, track, MediaKind.VIDEO, releaseSource); @@ -67,6 +72,30 @@ public Subscription onFrame(Consumer handler) { }; } + @Override + public VideoContentHint contentHint() { + return contentHint; + } + + @Override + public synchronized void setContentHint(VideoContentHint hint) { + requireNonNull(hint, "hint"); + + if (!isLive()) { + return; + } + + videoTrack().setContentHint(hint); + contentHint = hint; + } + + @Override + void adopt(MediaStreamTrack next) { + super.adopt(next); + + ((VideoTrack) next).setContentHint(contentHint); + } + @Override void beforeRelease() { List attached; diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/internal/SinkRegistry.java b/webrtc/src/main/java/dev/onvoid/webrtc/internal/SinkRegistry.java index dfd8458fe..d4722c2db 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/internal/SinkRegistry.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/internal/SinkRegistry.java @@ -208,7 +208,12 @@ public static void awaitDeferred() throws InterruptedException { done.await(); } - private static boolean inCallback() { + /** + * @return true while the current thread hands media to a sink, of any + * track, and holds the lock of WebRTC's broadcaster that adding a sink + * takes. + */ + public static boolean inCallback() { return CALLBACK_DEPTH.get()[0] > 0; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoContentHint.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoContentHint.java new file mode 100644 index 000000000..fc1eff52c --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoContentHint.java @@ -0,0 +1,60 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +/** + * What a video track shows, which decides what its senders keep when the + * network or the CPU cannot keep up: the frame rate or the resolution. See + * MediaStreamTrack Content + * Hints. + *

+ * A {@link dev.onvoid.webrtc.RTCRtpSendParameters#degradationPreference + * degradation preference} set on a sender takes precedence over the hint in + * that choice. + * + * @author Alex Andres + * + * @see VideoTrack#setContentHint(VideoContentHint) + */ +public enum VideoContentHint { + + /** + * No hint: the source decides. Screen shares keep the resolution, other + * video the frame rate. + */ + DEFAULT, + + /** + * Motion, such as a camera, a video or an animation: keep the frame rate, + * and lower the resolution where necessary. + */ + MOTION, + + /** + * Detail, such as drawings, slides or photos: keep the resolution, and + * lower the frame rate where necessary. + */ + DETAIL, + + /** + * Text, such as source code, a terminal or tables: keep the resolution, + * as with {@link #DETAIL}, which WebRTC treats it as for now, with every + * codec. No hint keeps text readable at any bitrate. + */ + TEXT + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrack.java index 6ad22e4c0..f064537aa 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrack.java @@ -108,6 +108,47 @@ public void removeSink(VideoTrackSink sink) { sinks.remove(sink); } + /** + * Returns what the track is hinted to show. + * + * @return the content hint, {@link VideoContentHint#DEFAULT} unless set. + * + * @throws IllegalStateException if the track has been disposed. + */ + public native VideoContentHint getContentHint(); + + /** + * Hints what the track shows, which decides what its senders keep when + * the network or the CPU cannot keep up: the frame rate or the + * resolution. Can be changed while the track is sent, without + * renegotiation; every sender of the track has taken it once this + * returns, and a sender the track is given later takes it as well. A + * change may make a sender reconfigure its encoder. Waits briefly for a + * WebRTC thread. + * + * @param hint What the track shows. + * + * @throws IllegalStateException if the track has been disposed, or if + * called from within the callback of a + * sink, of this track or any other. + */ + public void setContentHint(VideoContentHint hint) { + if (isNull(hint)) { + throw new NullPointerException(); + } + if (SinkRegistry.inCallback()) { + // A sender reconfigures on WebRTC's worker thread, which then adds + // its sink to the source again and waits for the lock this thread + // holds while it hands out the frame. + throw new IllegalStateException("The content hint cannot be set on a thread that is handing " + + "media to a sink. Set it on another thread"); + } + + setContentHintInternal(hint); + } + + private native void setContentHintInternal(VideoContentHint hint); + private native long addSinkInternal(VideoTrackSink sink, boolean shared); private native void removeSinkInternal(long sinkHandle); diff --git a/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/jni-config.json b/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/jni-config.json index 162229a11..f5120de5a 100644 --- a/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/jni-config.json +++ b/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/jni-config.json @@ -1143,6 +1143,12 @@ "allDeclaredMethods": true, "allDeclaredFields": true }, + { + "name": "dev.onvoid.webrtc.media.video.VideoContentHint", + "allDeclaredConstructors": true, + "allDeclaredMethods": true, + "allDeclaredFields": true + }, { "name": "dev.onvoid.webrtc.media.video.VideoDesktopSource", "allDeclaredConstructors": true, diff --git a/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/reflect-config.json b/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/reflect-config.json index c247003b7..8a6b335ba 100644 --- a/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/reflect-config.json +++ b/webrtc/src/main/resources/META-INF/native-image/dev.onvoid.webrtc/webrtc-java/reflect-config.json @@ -727,6 +727,10 @@ "name": "dev.onvoid.webrtc.media.video.VideoCaptureCapability", "queryAllPublicMethods": true }, + { + "name": "dev.onvoid.webrtc.media.video.VideoContentHint", + "queryAllPublicMethods": true + }, { "name": "dev.onvoid.webrtc.media.video.VideoDesktopSource", "queryAllPublicMethods": true @@ -1357,6 +1361,9 @@ { "name": "dev.onvoid.webrtc.media.video.VideoCaptureCapability[]" }, + { + "name": "dev.onvoid.webrtc.media.video.VideoContentHint[]" + }, { "name": "dev.onvoid.webrtc.media.video.VideoDesktopSource[]" }, diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java index 37dd7bb24..13e8c40a2 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import dev.onvoid.webrtc.RTCStats; @@ -49,7 +50,9 @@ import dev.onvoid.webrtc.media.audio.CustomAudioSource; import dev.onvoid.webrtc.media.video.CustomVideoSource; import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoContentHint; import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; import dev.onvoid.webrtc.media.video.desktop.DesktopSource; import org.junit.jupiter.api.AfterEach; @@ -410,6 +413,87 @@ void pushedVideoArrives() throws Exception { assertEquals(240, await(preview)[1]); } + @Test + void contentHintChangesWhileSending() throws Exception { + SignalingPair link = Signaling.loopback(); + AtomicInteger offers = new AtomicInteger(); + Signaling counted = new Signaling() { + + @Override + public void send(SignalMessage message) { + if (message.type() == SignalMessage.Type.OFFER) { + offers.incrementAndGet(); + } + link.first().send(message); + } + + @Override + public Subscription attach(Consumer inbound) { + return link.first().attach(inbound); + } + }; + Peer caller = rtc.createPeer(PeerOptions.builder().name("caller").signaling(counted).build()); + Peer callee = rtc.createPeer(PeerOptions.builder().name("callee").signaling(link.second()) + .polite(true).build()); + + CustomVideo video = rtc.media().customVideo("video"); + AtomicInteger tracks = new AtomicInteger(); + AtomicReference frames = new AtomicReference<>(new CountDownLatch(5)); + + callee.onTrack(track -> { + tracks.incrementAndGet(); + track.asVideo().onFrame(frame -> frames.get().countDown()); + }); + + video.setContentHint(VideoContentHint.TEXT); + caller.addTrack(video); + feedVideo(video, 320, 240); + + assertTrue(frames.get().await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + int offered = offers.get(); + + for (VideoContentHint hint : new VideoContentHint[] { VideoContentHint.MOTION, VideoContentHint.DEFAULT }) { + video.setContentHint(hint); + + assertEquals(hint, video.contentHint()); + assertEquals(hint, ((VideoTrack) ((LocalTrackImpl) video).nativeTrack()).getContentHint()); + + // The senders took the hint when the call returned, and the same + // remote track goes on receiving. + frames.set(new CountDownLatch(5)); + assertTrue(frames.get().await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + + assertEquals(offered, offers.get()); + assertEquals(1, tracks.get()); + assertEquals(1, callee.remoteTracks().size()); + + video.close(); + video.setContentHint(VideoContentHint.TEXT); + + assertEquals(VideoContentHint.DEFAULT, video.contentHint()); + } + + @Test + void replacedTrackKeepsContentHint() throws Exception { + // As a camera switch does it. + LocalVideoImpl video = (LocalVideoImpl) rtc.media().customVideo("video"); + CustomVideoSource nextSource = new CustomVideoSource(); + VideoTrack next = rtc.factory.createVideoTrack("next", nextSource); + + video.setContentHint(VideoContentHint.DETAIL); + video.setEnabled(false); + + LocalTrackImpl.Replaced replaced = video.replaceNative(next, nextSource::dispose); + + assertEquals(VideoContentHint.DETAIL, next.getContentHint()); + assertFalse(next.isEnabled()); + + replaced.track.dispose(); + replaced.release.run(); + await(video.closeAsync()); + } + @Test void slowFrameHandlerGetsNewest() throws Exception { Peer[] peers = pair(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoTrackTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoTrackTests.java index 74588377a..512322285 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoTrackTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoTrackTests.java @@ -61,6 +61,46 @@ void disableEnableTrack() { assertTrue(videoTrack.isEnabled()); } + @Test + void contentHintSet() { + assertEquals(VideoContentHint.DEFAULT, videoTrack.getContentHint()); + + for (VideoContentHint hint : VideoContentHint.values()) { + videoTrack.setContentHint(hint); + + assertEquals(hint, videoTrack.getContentHint()); + } + + assertThrows(NullPointerException.class, () -> videoTrack.setContentHint(null)); + } + + @Test + void contentHintRefusedInSink() { + CustomVideoSource source = new CustomVideoSource(); + VideoTrack track = factory.createVideoTrack("hinted", source); + AtomicReference refused = new AtomicReference<>(); + + // A sender would wait for the lock this thread holds in the sink. + track.addSink(frame -> { + frame.release(); + + try { + track.setContentHint(VideoContentHint.TEXT); + } + catch (Throwable e) { + refused.compareAndSet(null, e); + } + }); + + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> pushFrame(source)); + + assertInstanceOf(IllegalStateException.class, refused.get()); + assertEquals(VideoContentHint.DEFAULT, track.getContentHint()); + + track.dispose(); + source.dispose(); + } + @Test void addNullSink() { assertThrows(NullPointerException.class, () -> videoTrack.addSink(null));