diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 60783c55e..2efc28e22 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -351,6 +351,27 @@ A live stream has no end and no length: - Pausing stops reading from the server, but the server keeps sending. Resuming carries on from where playback paused, which leaves it behind real time by as long as it was paused. - To get back to live, close the source and open it again. +### Video Size of a Camera + +Many cameras say how large their pictures are only with each keyframe. FFmpeg reads 5 seconds of a source when it opens it. If the first keyframe comes later, `MediaInfo` reports a video size of 0x0. The video plays all the same, and its frames have their size. + +To know the size when the source opens, let FFmpeg read for longer than the camera's keyframe interval: + +```java +// Import required classes +import dev.onvoid.webrtc.media.player.MediaReader; +import dev.onvoid.webrtc.media.player.MediaReaderOptions; + +MediaReaderOptions options = MediaReaderOptions.builder() + .analyzeDuration(Duration.ofSeconds(15)) + .build(); + +MediaReader reader = new MediaReader("rtsp://camera.local/stream1", options); +MediaFileSource source = new MediaFileSource(reader, false); +``` + +The source takes over the reader and closes it. Opening takes up to the analysis duration on top of the timeout, as the camera sends in real time. FFmpeg also stops after 5 MB. For a stream of a high bit rate, raise that limit with `probeSize(bytes)` as well. + ## Closing Closing a `MediaFileSource` stops playback, and releases the player along with both media sources. Before that, release the senders a peer connection handed out, and the tracks: diff --git a/webrtc-media/src/main/cpp/include/JNI_MediaReader.h b/webrtc-media/src/main/cpp/include/JNI_MediaReader.h index f26d0b8ae..ed10c7f90 100644 --- a/webrtc-media/src/main/cpp/include/JNI_MediaReader.h +++ b/webrtc-media/src/main/cpp/include/JNI_MediaReader.h @@ -25,10 +25,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_player_MediaReader * Method: open - * Signature: (Ljava/lang/String;J)J + * Signature: (Ljava/lang/String;JJJ)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaReader_open - (JNIEnv *, jclass, jstring, jlong); + (JNIEnv *, jclass, jstring, jlong, jlong, jlong); /* * Class: dev_onvoid_webrtc_media_player_MediaReader diff --git a/webrtc-media/src/main/cpp/include/media/MediaReader.h b/webrtc-media/src/main/cpp/include/media/MediaReader.h index 0373f8e81..553e8c092 100644 --- a/webrtc-media/src/main/cpp/include/media/MediaReader.h +++ b/webrtc-media/src/main/cpp/include/media/MediaReader.h @@ -50,6 +50,13 @@ namespace ffmpeg MediaReader(const MediaReader &) = delete; MediaReader & operator=(const MediaReader &) = delete; + // How much of the source libavformat reads when it opens it, to + // find what the streams hold: in microseconds of the source and + // in bytes. 0 leaves its default. Opening may take the analysis + // duration on top of the timeout, which a live stream needs to + // send that much. Call this before Open. + void SetAnalysis(int64_t analyze_duration_us, int64_t probe_size); + // Opens the given source and selects the video and audio stream // that are meant to be played. Returns 0, or the negative AVERROR // libavformat reported, in which case the reader stays closed. @@ -134,8 +141,9 @@ namespace ffmpeg // makes the blocking call fail with AVERROR_EXIT. static int OnInterrupt(void * opaque); - // Starts the time the next blocking call has, and ends it. - void BeginBlocking(); + // Starts the time the next blocking call has, the timeout and the + // given extra time, and ends it. + void BeginBlocking(int64_t extra_us = 0); void EndBlocking(); // A blocking call that failed because it ran out of time rather @@ -149,6 +157,10 @@ namespace ffmpeg const int64_t timeout_us_; + // 0 for the defaults of libavformat. + int64_t analyze_duration_us_ = 0; + int64_t probe_size_ = 0; + // When the current blocking call runs out of time, on the clock // of av_gettime_relative(), or 0 while none is running. std::atomic deadline_us_{ 0 }; diff --git a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h index 5f9d566a9..37a54cb8d 100644 --- a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h @@ -177,6 +177,11 @@ namespace ffmpeg // for a key frame, which it is after it lost pictures it needs. bool SkipUntilKeyFrame(const AVPacket * packet); + // Whether an error of the codec only says that the stream started + // between key frames: one before the first picture, which is + // passed over. + bool IsLeadingError(int result) const; + void KeepPacket(const AVPacket * packet); void DropKeptPackets(); void DropPending(); @@ -233,6 +238,10 @@ namespace ffmpeg // and starts again at a key frame. bool awaiting_key_frame_ = false; size_t skipped_ = 0; + + // Whether the codec has given out a picture since the decoder + // was opened. + bool has_pictures_ = false; std::deque pending_; // Where the frames after a seek are to start. diff --git a/webrtc-media/src/main/cpp/src/JNI_MediaReader.cpp b/webrtc-media/src/main/cpp/src/JNI_MediaReader.cpp index 66affad4b..da062f6a1 100644 --- a/webrtc-media/src/main/cpp/src/JNI_MediaReader.cpp +++ b/webrtc-media/src/main/cpp/src/JNI_MediaReader.cpp @@ -62,7 +62,8 @@ namespace } JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaReader_open -(JNIEnv * env, jclass caller, jstring source, jlong timeoutUs) +(JNIEnv * env, jclass caller, jstring source, jlong timeoutUs, jlong analyzeDurationUs, + jlong probeSize) { return ffmpeg::CallFromJava(env, 0, [&]() -> jlong { if (source == nullptr) { @@ -92,6 +93,9 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaReader_open } auto reader = std::make_unique(static_cast(timeoutUs)); + + reader->SetAnalysis(static_cast(analyzeDurationUs), static_cast(probeSize)); + int result = reader->Open(url); if (result < 0) { diff --git a/webrtc-media/src/main/cpp/src/media/MediaReader.cpp b/webrtc-media/src/main/cpp/src/media/MediaReader.cpp index e1a2a1608..abea180f4 100644 --- a/webrtc-media/src/main/cpp/src/media/MediaReader.cpp +++ b/webrtc-media/src/main/cpp/src/media/MediaReader.cpp @@ -57,6 +57,12 @@ namespace ffmpeg Close(); } + void MediaReader::SetAnalysis(int64_t analyze_duration_us, int64_t probe_size) + { + analyze_duration_us_ = std::max(analyze_duration_us, 0); + probe_size_ = std::max(probe_size, 0); + } + int MediaReader::Open(const std::string & url) { Close(); @@ -76,6 +82,15 @@ namespace ffmpeg format_context_->interrupt_callback.callback = &MediaReader::OnInterrupt; format_context_->interrupt_callback.opaque = this; + // Set on the context rather than passed as options, which only fail + // where an allocation can. 0 is what libavformat takes for its own + // choice of analysis duration; its probe size has a default instead. + format_context_->max_analyze_duration = analyze_duration_us_; + + if (probe_size_ > 0) { + format_context_->probesize = probe_size_; + } + AVDictionary * options = nullptr; int result = AllocationFault::CheckOption( @@ -105,7 +120,9 @@ namespace ffmpeg return TranslateError(result); } - BeginBlocking(); + // A live stream takes as long to send what is analyzed as it lasts, + // which the timeout alone may not cover. + BeginBlocking(analyze_duration_us_); result = avformat_find_stream_info(format_context_, nullptr); EndBlocking(); @@ -382,7 +399,7 @@ namespace ffmpeg return 0; } - void MediaReader::BeginBlocking() + void MediaReader::BeginBlocking(int64_t extra_us) { timed_out_.store(false); @@ -394,8 +411,9 @@ namespace ffmpeg // Saturated: a timeout too long to count arrives as INT64_MAX, and the // sum would wrap around into a deadline that has passed already. const int64_t now = av_gettime_relative(); + const int64_t time_us = extra_us > INT64_MAX - timeout_us_ ? INT64_MAX : timeout_us_ + extra_us; - deadline_us_.store(timeout_us_ > INT64_MAX - now ? INT64_MAX : now + timeout_us_); + deadline_us_.store(time_us > INT64_MAX - now ? INT64_MAX : now + time_us); } void MediaReader::EndBlocking() diff --git a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp index 29025d163..5341199e8 100644 --- a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -342,6 +342,7 @@ namespace ffmpeg draining_ = false; awaiting_key_frame_ = false; skipped_ = 0; + has_pictures_ = false; discarding_ = false; time_base_ = { 0, 1 }; start_time_us_ = 0; @@ -439,9 +440,25 @@ namespace ffmpeg } } + if (packet != nullptr && IsLeadingError(result)) { + // The packet is taken all the same: the codec has it, or it is + // one of those that cannot be decoded. + return 0; + } + return result; } + bool VideoDecoder::IsLeadingError(int result) const + { + // A live stream joined between key frames, as an IP camera that sends + // its parameter sets only in band is: what comes before the first key + // frame cannot be decoded, and is no fault of the stream. Passing + // over it starts decoding at that key frame. Once there are pictures, + // such an error is a fault. + return result == AVERROR_INVALIDDATA && !has_pictures_; + } + int VideoDecoder::ReceiveFrame(AVFrame ** frame, int64_t * timestamp_us) { if (codec_context_ == nullptr || decoded_ == nullptr) { @@ -479,9 +496,18 @@ namespace ffmpeg int result = ReceivePicture(frame, timestamp_us); + if (IsLeadingError(result)) { + // With frame threads the codec tells of a packet it could not + // decode on the way out, and the pictures behind it are still + // to come. + continue; + } if (result < 0) { return result; } + + has_pictures_ = true; + if (*frame == nullptr) { // Dropped before it was read back and converted. dropped_before_ = true; diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java index a26629925..388f5649b 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.file.Path; import java.time.Duration; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -162,6 +163,27 @@ public MediaFileSource(String source, Duration timeout, boolean hardwareDecoding hardwareDecoding); } + /** + * Plays the source of a reader opened by the caller, such as one given + * {@link MediaReaderOptions}, decoding its video in hardware where the + * platform can if asked to, as {@link MediaPlayer#MediaPlayer(MediaReader, + * CustomVideoSource, CustomAudioSource, boolean)} describes. + *

+ * The reader is taken over, and closed with this source, or right away + * if this source cannot be made. It must not be used afterwards. + * + * @param reader An open reader of the source to play. + * @param hardwareDecoding Whether to decode video in hardware if possible. + * HEVC is decoded in hardware regardless. + * + * @throws IOException if the source cannot be decoded. + * @throws IllegalStateException if the reader is closed. + */ + public MediaFileSource(MediaReader reader, boolean hardwareDecoding) throws IOException { + this(Objects.requireNonNull(reader, "Reader is null"), CustomVideoSource::new, + CustomAudioSource::new, hardwareDecoding); + } + /** * Builds on a reader that is open, which this source takes over whether * or not it is made. The suppliers make the media sources, which tests diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaInfo.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaInfo.java index 9edd809f1..5ecb0fe2e 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaInfo.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaInfo.java @@ -23,6 +23,12 @@ * A stream that is absent is reported as a zero size, frame rate, sample rate * or channel count, so {@link #hasVideo()} and {@link #hasAudio()} are the * questions to ask before reading the rest. + *

+ * A video stream can be there with a zero size, too: a live stream that + * carries its parameter sets only in band, as many IP cameras do, says how + * large its pictures are only with its first keyframe. If that comes later + * than FFmpeg looks for it when the source is opened, the size is not known + * yet. The video plays all the same, and its frames have their size. * * @author Alex Andres */ @@ -31,10 +37,10 @@ public class MediaInfo { /** How long the source runs in microseconds, or 0 if it is not known. */ private final long durationUs; - /** The width of the video as it is shown, or 0 if there is no video. */ + /** The width of the video as it is shown, or 0 if there is none or it is not known. */ private final int videoWidth; - /** The height of the video as it is shown, or 0 if there is no video. */ + /** The height of the video as it is shown, or 0 if there is none or it is not known. */ private final int videoHeight; /** The frame rate of the video, or 0 if there is no video. */ @@ -90,12 +96,16 @@ public long getDurationUs() { } /** - * Returns whether the source has a video stream that can be played. + * Returns whether the source has a video stream that can be played. Its + * size may not be known yet, see the class description. * * @return True if there is video. */ public boolean hasVideo() { - return videoWidth > 0 && videoHeight > 0; + // The codec, not the size: native code names one for every video + // stream it plays, while the size can be missing until the first + // keyframe. + return videoCodec != null; } /** @@ -103,7 +113,8 @@ public boolean hasVideo() { * turned upright. A video stored sideways reports its stored height here, * while its frames keep their stored size and carry the rotation. * - * @return The width, or {@code 0} if there is no video. + * @return The width, or {@code 0} if there is no video or its size is not + * known yet. */ public int getVideoWidth() { return videoWidth; @@ -114,7 +125,8 @@ public int getVideoWidth() { * turned upright. A video stored sideways reports its stored width here, * while its frames keep their stored size and carry the rotation. * - * @return The height, or {@code 0} if there is no video. + * @return The height, or {@code 0} if there is no video or its size is not + * known yet. */ public int getVideoHeight() { return videoHeight; diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReader.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReader.java index 5fd2052f8..e3a4cee77 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReader.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReader.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.file.Path; import java.time.Duration; +import java.util.Objects; import dev.onvoid.webrtc.internal.Timeouts; @@ -96,11 +97,30 @@ public MediaReader(String source) throws IOException { * @throws IllegalArgumentException if the timeout is negative. */ public MediaReader(String source, Duration timeout) throws IOException { + this(source, MediaReaderOptions.builder().timeout(timeout).build()); + } + + /** + * Opens the given media source with the given options: the timeout of + * each operation on it, and how much of it FFmpeg reads to find out what + * its streams hold. + * + * @param source The path or {@code rtsp://} URL of the source to open. + * @param options How to open the source. + * + * @throws IOException if the source cannot be opened in time, or holds + * nothing that can be played. + */ + public MediaReader(String source, MediaReaderOptions options) throws IOException { + Objects.requireNonNull(options, "Options are null"); + // Zero is the one timeout without limit: a positive one rounds up to a // microsecond, and one too long to count is as long as can be counted. - Timeouts.requireNonNegative(timeout, "timeout"); + Duration analyzeDuration = options.getAnalyzeDuration(); - handle = open(source, Timeouts.toMicros(timeout)); + handle = open(source, Timeouts.toMicros(options.getTimeout()), + analyzeDuration != null ? Timeouts.toMicros(analyzeDuration) : 0, + options.getProbeSize()); } /** @@ -149,7 +169,8 @@ synchronized long detach() { return detaching; } - private static native long open(String source, long timeoutUs) throws IOException; + private static native long open(String source, long timeoutUs, long analyzeDurationUs, + long probeSize) throws IOException; private static native MediaInfo info(long handle); diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReaderOptions.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReaderOptions.java new file mode 100644 index 000000000..366a69dd0 --- /dev/null +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaReaderOptions.java @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.player; + +import java.time.Duration; + +import dev.onvoid.webrtc.internal.Timeouts; + +/** + * How a {@link MediaReader} opens its source: how long each operation on it + * may wait, and how much of it FFmpeg reads to find out what its streams + * hold. + *

{@code
+ * MediaReaderOptions options = MediaReaderOptions.builder()
+ *         .analyzeDuration(Duration.ofSeconds(15))
+ *         .build();
+ *
+ * MediaReader reader = new MediaReader("rtsp://camera.local/stream1", options);
+ * }
+ *

+ * The analysis matters for live streams. FFmpeg reads 5 seconds of a source, + * and up to 5 MB, to learn the size of its video, among others. A camera that + * says how large its pictures are only with each keyframe, and sends one less + * often than that, leaves the size unknown; {@link MediaInfo} then reports + * 0x0. A longer analysis finds it, and takes that much longer to open a live + * stream, which sends in real time. + * + * @author Alex Andres + */ +public final class MediaReaderOptions { + + private final Duration timeout; + + private final Duration analyzeDuration; + + private final long probeSize; + + + private MediaReaderOptions(Builder builder) { + this.timeout = builder.timeout; + this.analyzeDuration = builder.analyzeDuration; + this.probeSize = builder.probeSize; + } + + /** + * Returns a builder with the default options: a timeout of + * {@link MediaReader#DEFAULT_TIMEOUT}, and the analysis FFmpeg chooses. + * + * @return A builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * @return How long opening, and later any single read, seek or close, may + * wait on the source. {@link Duration#ZERO} waits as long as it takes. + */ + public Duration getTimeout() { + return timeout; + } + + /** + * @return How much of the source FFmpeg reads to find out what its streams + * hold, or {@code null} for the time FFmpeg chooses. + */ + public Duration getAnalyzeDuration() { + return analyzeDuration; + } + + /** + * @return How many bytes of the source FFmpeg reads at most to find out + * what its streams hold, or 0 for its default. + */ + public long getProbeSize() { + return probeSize; + } + + @Override + public String toString() { + return "MediaReaderOptions[timeout=" + timeout + ", analyzeDuration=" + analyzeDuration + + ", probeSize=" + probeSize + "]"; + } + + + /** + * Collects the options of a reader. Each option not set keeps its default. + */ + public static final class Builder { + + /** + * The least FFmpeg reads of a source, in bytes, as smaller amounts + * cannot tell a format. + */ + private static final long MIN_PROBE_SIZE = 32; + + private Duration timeout = MediaReader.DEFAULT_TIMEOUT; + + private Duration analyzeDuration; + + private long probeSize; + + + private Builder() { + } + + /** + * Sets how long opening, and later any single read, seek or close, + * may wait on the source. + * + * @param timeout {@link Duration#ZERO} waits as long as it takes; a + * positive timeout waits at least a microsecond. + * + * @return This builder. + * + * @throws IllegalArgumentException if the timeout is negative. + */ + public Builder timeout(Duration timeout) { + this.timeout = Timeouts.requireNonNegative(timeout, "timeout"); + return this; + } + + /** + * Sets how much of the source FFmpeg reads to find out what its + * streams hold, for one whose keyframes come further apart than the 5 + * seconds it reads otherwise. Opening a live stream may take this + * long on top of the timeout, as the stream sends in real time. + * + * @param duration More than 0. + * + * @return This builder. + * + * @throws IllegalArgumentException if the duration is not positive. + */ + public Builder analyzeDuration(Duration duration) { + this.analyzeDuration = Timeouts.requirePositive(duration, "analyze duration"); + return this; + } + + /** + * Sets how many bytes of the source FFmpeg reads at most to find out + * what its streams hold, which is 5,000,000 otherwise. Raise it with + * the {@link #analyzeDuration(Duration) analysis duration} for a + * stream of a high bit rate, which may reach this first. + * + * @param bytes At least 32. + * + * @return This builder. + * + * @throws IllegalArgumentException if the size is less than 32. + */ + public Builder probeSize(long bytes) { + if (bytes < MIN_PROBE_SIZE) { + throw new IllegalArgumentException("The probe size must be at least " + + MIN_PROBE_SIZE + " bytes, got " + bytes); + } + + this.probeSize = bytes; + return this; + } + + /** + * @return The options. + */ + public MediaReaderOptions build() { + return new MediaReaderOptions(this); + } + } + +} diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaFileSourceTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaFileSourceTest.java index 685051eb2..caa58ec8e 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaFileSourceTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaFileSourceTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -30,6 +31,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import dev.onvoid.webrtc.PeerConnectionFactory; import dev.onvoid.webrtc.media.audio.AudioDeviceModule; @@ -44,6 +46,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; @@ -59,6 +62,12 @@ class MediaFileSourceTest { private static final String ASSET = "/media-test.webm"; + /** H.264 with its parameter sets in avcC form, the CodecPrivate. */ + private static final String H264_ASSET = "/media-test-h264.mkv"; + + @TempDir + Path tempDir; + private AudioDeviceModule audioModule; private PeerConnectionFactory factory; @@ -129,6 +138,95 @@ public void onEndOfStream() { } } + @Test + void sendsVideoWhoseSizeComesWithTheFirstKeyframe() throws Exception { + Path original = asset(H264_ASSET); + int width; + int height; + + try (MediaReader reader = new MediaReader(original)) { + width = reader.getInfo().getVideoWidth(); + height = reader.getInfo().getVideoHeight(); + } + + // As an IP camera sends it: parameter sets in band only, and the first + // keyframe after more time than FFmpeg looks for the size. + Path file = TestMedia.inBandParameterSetsMatroska(tempDir, original, 6000); + + AtomicInteger frames = new AtomicInteger(); + AtomicInteger frameWidth = new AtomicInteger(); + AtomicInteger frameHeight = new AtomicInteger(); + AtomicReference error = new AtomicReference<>(); + CountDownLatch ended = new CountDownLatch(1); + + try (MediaFileSource source = new MediaFileSource(file)) { + MediaInfo info = source.getInfo(); + + // The size is not known yet, which is what this is about. + assertEquals(0, info.getVideoWidth(), info.toString()); + assertTrue(info.hasVideo(), info.toString()); + assertEquals("h264", info.getVideoCodec()); + assertNotNull(source.getVideoSource()); + + VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource()); + VideoTrackSink videoSink = frame -> { + frames.incrementAndGet(); + frameWidth.set(frame.buffer.getWidth()); + frameHeight.set(frame.buffer.getHeight()); + }; + + videoTrack.addSink(videoSink); + + source.setListener(new MediaPlayerListener() { + + @Override + public void onEndOfStream() { + ended.countDown(); + } + + @Override + public void onError(String message) { + error.set(message); + ended.countDown(); + } + }); + + source.play(); + + assertTrue(ended.await(30, TimeUnit.SECONDS), "no end of stream"); + // What comes before the keyframe is passed over, not an error. + assertNull(error.get()); + + // 30 pictures in two groups of 15 follow the keyframe: losing the + // first group to the pictures before it would leave 16. + assertTrue(frames.get() > 20, "video frames: " + frames.get()); + assertEquals(width, frameWidth.get()); + assertEquals(height, frameHeight.get()); + + videoTrack.removeSink(videoSink); + videoTrack.dispose(); + } + } + + @Test + void playsReaderOfTheCaller() throws Exception { + Path file = TestMedia.inBandParameterSetsMatroska(tempDir, asset(H264_ASSET), 6000); + MediaReaderOptions options = MediaReaderOptions.builder() + .analyzeDuration(Duration.ofSeconds(10)) + .build(); + MediaReader reader = new MediaReader(file.toString(), options); + + try (MediaFileSource source = new MediaFileSource(reader, false)) { + // What the options found. + assertEquals(320, source.getInfo().getVideoWidth()); + assertNotNull(source.getVideoSource()); + } + + // Taken over, and closed with the source. + assertThrows(IllegalStateException.class, reader::getInfo); + assertThrows(IllegalStateException.class, () -> new MediaFileSource(reader, false)); + } + @Test void missingFileFails() { assertThrows(IOException.class, () -> new MediaFileSource(Paths.get("no-such-file.webm"))); @@ -240,9 +338,13 @@ void closesFromSinkWhileClosing() throws Exception { } private static Path asset() throws Exception { - URL url = MediaFileSourceTest.class.getResource(ASSET); + return asset(ASSET); + } + + private static Path asset(String name) throws Exception { + URL url = MediaFileSourceTest.class.getResource(name); - assertNotNull(url, "Test asset " + ASSET + " is missing"); + assertNotNull(url, "Test asset " + name + " is missing"); return Paths.get(url.toURI()); } diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaReaderTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaReaderTest.java index a0e228579..84e31b2cd 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaReaderTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaReaderTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -115,6 +116,76 @@ void opensWithLongestTimeout() throws Exception { } } + @Test + void opensWithLongestTimeoutAndAnalysis() throws Exception { + // The analysis is added to the timeout of opening, and the sum of + // the two saturates as well. + MediaReaderOptions options = MediaReaderOptions.builder() + .timeout(Duration.ofSeconds(Long.MAX_VALUE)) + .analyzeDuration(Duration.ofSeconds(Long.MAX_VALUE)) + .build(); + + try (MediaReader reader = new MediaReader(asset().toAbsolutePath().toString(), options)) { + assertEquals(320, reader.getInfo().getVideoWidth()); + } + } + + @Test + void longerAnalysisFindsLateSize(@TempDir Path dir) throws Exception { + // The first keyframe, which says how large the pictures are, comes + // after six seconds: later than FFmpeg looks by default. + Path file = TestMedia.inBandParameterSetsMatroska(dir, h264Asset(), 6000); + + try (MediaReader reader = new MediaReader(file)) { + assertEquals(0, reader.getInfo().getVideoWidth()); + } + + MediaReaderOptions options = MediaReaderOptions.builder() + .analyzeDuration(Duration.ofSeconds(10)) + .build(); + + try (MediaReader reader = new MediaReader(file.toString(), options)) { + MediaInfo info = reader.getInfo(); + + assertEquals(320, info.getVideoWidth()); + assertEquals(240, info.getVideoHeight()); + } + } + + @Test + void probeSizeLimitsAnalysis(@TempDir Path dir) throws Exception { + Path file = TestMedia.inBandParameterSetsMatroska(dir, h264Asset(), 6000); + + // Long enough to reach the keyframe, but not in so few bytes. + MediaReaderOptions options = MediaReaderOptions.builder() + .analyzeDuration(Duration.ofSeconds(10)) + .probeSize(1000) + .build(); + + try (MediaReader reader = new MediaReader(file.toString(), options)) { + assertEquals(0, reader.getInfo().getVideoWidth()); + } + } + + @Test + void rejectsInvalidOptions() { + MediaReaderOptions.Builder builder = MediaReaderOptions.builder(); + + assertThrows(IllegalArgumentException.class, () -> builder.timeout(Duration.ofNanos(-1))); + assertThrows(IllegalArgumentException.class, () -> builder.analyzeDuration(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> builder.analyzeDuration(Duration.ofNanos(-1))); + assertThrows(IllegalArgumentException.class, () -> builder.probeSize(31)); + assertThrows(NullPointerException.class, () -> builder.timeout(null)); + assertThrows(NullPointerException.class, () -> builder.analyzeDuration(null)); + assertThrows(NullPointerException.class, () -> new MediaReader(asset().toString(), (MediaReaderOptions) null)); + + MediaReaderOptions defaults = builder.build(); + + assertEquals(MediaReader.DEFAULT_TIMEOUT, defaults.getTimeout()); + assertNull(defaults.getAnalyzeDuration()); + assertEquals(0, defaults.getProbeSize()); + } + @Test void readsAviInfo() throws Exception { try (MediaReader reader = new MediaReader(asset(AVI_ASSET))) { @@ -295,6 +366,11 @@ private static Path asset() throws Exception { return asset(ASSET); } + /** 320x240 H.264 with its parameter sets in the CodecPrivate. */ + private static Path h264Asset() throws Exception { + return asset("/media-test-h264.mkv"); + } + static Path asset(String name) throws Exception { URL url = MediaReaderTest.class.getResource(name); diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java index 4d0af5f85..c28b489ed 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java @@ -26,7 +26,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.IntFunction; /** @@ -363,6 +365,109 @@ private static byte[] projection(double yaw, double roll) { ebml(0x7675, ByteBuffer.allocate(8).putDouble(roll).array())); } + /** + * Copies the H.264 video of a Matroska file the way a camera that sends + * its parameter sets only in band streams it: in Annex B, with no + * CodecPrivate and no size, and SPS and PPS only in front of each + * keyframe. The stream starts in the middle of a group of pictures, with + * pictures between keyframes for the given time, which cannot be decoded + * without the parameter sets. Once that is longer than FFmpeg looks for + * the size when it opens the file, the size is not known before playback. + * + * @param directory Where to write the file. + * @param source The file to copy, with H.264 in avcC form. + * @param leadMs How long the stream runs before its first keyframe. + * + * @return The file written. + */ + static Path inBandParameterSetsMatroska(Path directory, Path source, long leadMs) + throws IOException { + MatroskaContent content = new MatroskaContent(Files.readAllBytes(source)); + ByteBuffer avcC = ByteBuffer.wrap(content.codecPrivates.get((long) content.videoTrack)); + ByteArrayOutputStream parameterSets = new ByteArrayOutputStream(); + + // The configuration version, profile, compatibility and level, then + // the size of the NAL unit lengths. + avcC.position(4); + + int lengthSize = (avcC.get() & 0x03) + 1; + + for (int count = avcC.get() & 0x1F; count > 0; count--) { + writeNal(parameterSets, avcC, avcC.getShort() & 0xFFFF); + } + for (int count = avcC.get() & 0xFF; count > 0; count--) { + writeNal(parameterSets, avcC, avcC.getShort() & 0xFFFF); + } + + List video = new ArrayList<>(); + List pictures = new ArrayList<>(); + + for (Frame frame : content.frames) { + if (frame.track == content.videoTrack) { + video.add(frame); + + if (!frame.key) { + pictures.add(frame); + } + } + } + + List frames = new ArrayList<>(); + + for (int i = 0; i * 40L < leadMs; i++) { + Frame picture = pictures.get(i % pictures.size()); + + frames.add(new Frame(1, i * 40L, false, annexB(picture.data, lengthSize, null))); + } + for (Frame frame : video) { + frames.add(new Frame(1, leadMs + frame.timeMs, frame.key, + annexB(frame.data, lengthSize, frame.key ? parameterSets.toByteArray() : null))); + } + + byte[] track = ebml(0xAE, + ebmlUInt(0xD7, 1), // TrackNumber + ebmlUInt(0x73C5, 1), // TrackUID + ebmlUInt(0x83, 1), // TrackType, video + // DefaultDuration, 40 ms. Without a duration or a frame rate + // FFmpeg cannot tell how long it has looked, and reads on to + // the keyframe, where a live stream would have timing. + ebmlUInt(0x23E383, 40_000_000), + ebml(0x86, ascii("V_MPEG4/ISO/AVC"))); + + return Files.write(directory.resolve("in-band-parameter-sets.mkv"), + matroska(track, frames, 1, -1)); + } + + /** A sample of NAL units with lengths in front, in Annex B. */ + private static byte[] annexB(byte[] sample, int lengthSize, byte[] prefix) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteBuffer in = ByteBuffer.wrap(sample); + + if (prefix != null) { + writeBytes(out, prefix); + } + + while (in.hasRemaining()) { + int length = 0; + + for (int i = 0; i < lengthSize; i++) { + length = (length << 8) | (in.get() & 0xFF); + } + + writeNal(out, in, length); + } + + return out.toByteArray(); + } + + /** Writes a NAL unit read from the buffer, after a start code. */ + private static void writeNal(ByteArrayOutputStream out, ByteBuffer in, int length) { + writeBytes(out, new byte[] { 0, 0, 0, 1 }); + out.write(in.array(), in.position(), length); + + in.position(in.position() + length); + } + /** * Copies a Matroska file holding a video and an audio track so that its * video comes late. The audio is repeated until it runs for the given @@ -835,6 +940,8 @@ private static final class Frame { private static final class MatroskaContent { final List frames = new ArrayList<>(); + /** The CodecPrivate of each track that has one, by track number. */ + final Map codecPrivates = new HashMap<>(); byte[] tracks; int videoTrack; @@ -877,6 +984,9 @@ private void read(int end) { videoTrack = (int) trackNumber; } break; + case 0x63A2: // CodecPrivate, which follows the number + codecPrivates.put(trackNumber, Arrays.copyOfRange(data.array(), start, stop)); + break; case 0xE7: // Timestamp clusterTime = readUInt(stop); break;