diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 60783c55e..074e02293 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -371,7 +371,7 @@ The FFmpeg build is deliberately small, and carries only what this module plays: | --- | --- | | **Containers** | MP4 and MOV, Matroska and WebM, AVI, MPEG-TS, FLV, WAV, MP3, Ogg, FLAC, AAC | | **Video** | H.264, H.265/HEVC (hardware only), VP8, VP9, AV1, MPEG-4 (including Xvid and DivX), Microsoft MPEG-4 v1 to v3, MJPEG | -| **Audio** | AAC, MP3, MP2, AC-3, Opus, Vorbis, FLAC, PCM, MS and IMA ADPCM | +| **Audio** | AAC, MP3, MP2, AC-3, Opus, Vorbis, FLAC, PCM, G.711 (A-law and ยต-law), MS and IMA ADPCM | Audio of any rate or layout is resampled to what WebRTC takes: 48 kHz 16-bit PCM, in mono or stereo. Video that decodes to I420, which is almost all 8-bit H.264, VP8, VP9 and MPEG-4, reaches the encoder without being copied. Anything else is converted first. diff --git a/webrtc-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt b/webrtc-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt index 05dcdacb1..6f1ead5ea 100644 --- a/webrtc-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt +++ b/webrtc-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt @@ -82,6 +82,9 @@ set(FFMPEG_COMPONENTS --enable-decoder=pcm_s16be --enable-decoder=pcm_u8 --enable-decoder=pcm_f32le + # G.711, the audio most IP cameras send over RTSP. + --enable-decoder=pcm_alaw + --enable-decoder=pcm_mulaw # Audio common in AVI files and in MPEG transport streams. --enable-decoder=mp2 --enable-decoder=ac3 diff --git a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h index e4cda4d47..2a0cf7cb2 100644 --- a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h +++ b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h @@ -98,6 +98,11 @@ namespace ffmpeg // Returns 0 or a negative AVERROR. int Initialize(); + // Why Initialize failed, in words for an exception: which + // decoder is missing, where that is the reason. Empty if it has + // not failed. + const std::string & GetInitializeError() const; + void Play(); void Pause(); @@ -216,6 +221,7 @@ namespace ffmpeg bool hardware_decoding_ = false; bool has_video_ = false; bool has_audio_ = false; + std::string initialize_error_; std::thread thread_; std::thread event_thread_; diff --git a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h index 5f9d566a9..9f716c21c 100644 --- a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h @@ -74,6 +74,14 @@ namespace ffmpeg // Returns 0 or a negative AVERROR. int Open(const AVStream * stream, bool hardware = false, int64_t start_time_us = 0); + // Whether the codec is decoded on the hardware only, never in + // software: HEVC. FFmpeg's decoder of it is in the build all the + // same, because the hardware decoders run inside it (it reads the + // headers and hands the slices to the hardware), but it decodes no + // picture itself. Where the hardware cannot, opening or decoding + // the stream fails. + static bool IsHardwareOnly(AVCodecID id); + void Close(); // Drops everything the decoder holds, which is what a seek needs: diff --git a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp index 9f5744239..d0c007352 100644 --- a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp @@ -92,10 +92,16 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_create int result = player->Initialize(); if (result < 0) { + std::string reason = player->GetInitializeError(); + + if (reason.empty()) { + reason = ffmpeg::ErrorText(result); + } + // Released before throwing, see DiscardPlayer. player.reset(); - ffmpeg::ThrowIOException(env, "Opening the decoders failed: " + ffmpeg::ErrorText(result)); + ffmpeg::ThrowIOException(env, "Opening the decoders failed: " + reason); return 0; } diff --git a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp index cb42d1eef..7a802b909 100644 --- a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp @@ -25,9 +25,35 @@ #include extern "C" { +#include #include } +namespace +{ + // What an exception says when a decoder for the stream could not be + // opened. FFmpeg's own words for a missing decoder do not say which + // one, and with several streams the codec is what tells them apart. + std::string DescribeOpenError(const AVStream * stream, int error) + { + if (error != AVERROR_DECODER_NOT_FOUND) { + return ffmpeg::ErrorText(error); + } + + const AVCodecID id = stream->codecpar->codec_id; + const std::string name = avcodec_get_name(id); + + if (ffmpeg::VideoDecoder::IsHardwareOnly(id)) { + return "No hardware decoder for " + name + ", which is decoded in hardware only"; + } + + const char * type = av_get_media_type_string(stream->codecpar->codec_type); + + return std::string("No decoder for ") + (type != nullptr ? type : "unknown") + + " codec " + name; + } +} + namespace ffmpeg { MediaPlayer::MediaPlayer(std::unique_ptr reader, @@ -74,6 +100,8 @@ namespace ffmpeg reader_->GetStartTimeUs()); if (result < 0) { + initialize_error_ = DescribeOpenError(reader_->GetVideoStream(), result); + return result; } @@ -84,6 +112,8 @@ namespace ffmpeg int result = audio_decoder_.Open(reader_->GetAudioStream(), reader_->GetStartTimeUs()); if (result < 0) { + initialize_error_ = DescribeOpenError(reader_->GetAudioStream(), result); + return result; } @@ -113,6 +143,11 @@ namespace ffmpeg return 0; } + const std::string & MediaPlayer::GetInitializeError() const + { + return initialize_error_; + } + void MediaPlayer::Play() { std::lock_guard lock(mutex_); diff --git a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp index 29025d163..9c877ded0 100644 --- a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -57,16 +57,6 @@ namespace #endif } - // Whether the codec is decoded on the hardware only, never in software: - // HEVC. FFmpeg's decoder of it is in the build all the same, because the - // hardware decoders run inside it (it reads the headers and hands the - // slices to the hardware), but it decodes no picture itself. Where the - // hardware cannot, opening or decoding the stream fails. - bool IsHardwareOnly(AVCodecID id) - { - return id == AV_CODEC_ID_HEVC; - } - // The decoder to open for a codec. Software gets the one FFmpeg prefers. // Hardware gets it too, unless it has no hardware configuration: where // another decoder of the codec stands in front of the one that decodes on @@ -114,6 +104,11 @@ namespace ffmpeg Close(); } + bool VideoDecoder::IsHardwareOnly(AVCodecID id) + { + return id == AV_CODEC_ID_HEVC; + } + int VideoDecoder::Open(const AVStream * stream, bool hardware, int64_t start_time_us) { Close(); diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java index 43362062e..b966c26f7 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java @@ -222,7 +222,8 @@ void hevcWithoutHardwareDoesNotOpen() throws Exception { IOException error = assertThrows(IOException.class, () -> new MediaPlayer(new MediaReader(path(HEVC)), source, null, hardware)); - assertTrue(error.getMessage().contains("Decoder not found"), error.getMessage()); + assertTrue(error.getMessage().endsWith("No hardware decoder for hevc, which is decoded in hardware only"), + error.getMessage()); } finally { source.dispose(); diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java index b5756c67e..bf8aad5c0 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -1347,14 +1348,45 @@ void outlivesDisposedSources() throws Exception { @Test void failsWithoutDecoder() throws Exception { - Path file = TestMedia.muLawWav(tempDir); + Path file = TestMedia.pcm24Wav(tempDir); try (Sources sources = new Sources()) { // The reader opens, since the demuxer knows the format. MediaReader reader = new MediaReader(file); - assertThrows(IOException.class, + IOException error = assertThrows(IOException.class, () -> new MediaPlayer(reader, null, sources.audio)); + + // Names the codec, which the source may have several of. + assertEquals("Opening the decoders failed: No decoder for audio codec pcm_s24le", + error.getMessage()); + } + } + + @Test + void playsALaw() throws Exception { + assertPlaysG711(true); + } + + @Test + void playsMuLaw() throws Exception { + assertPlaysG711(false); + } + + private void assertPlaysG711(boolean aLaw) throws Exception { + // One second of 8 kHz mono, as an IP camera sends it. + Path file = TestMedia.g711Wav(tempDir, aLaw); + + try (Playback playback = new Playback(new MediaReader(file))) { + playback.player.play(); + + assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream"); + assertNull(playback.error.get()); + + assertEquals(48000, playback.sampleRate.get()); + assertEquals(1, playback.channels.get()); + assertEquals(100, playback.chunks.get()); + assertEquals(48000, playback.nonZeroSamples.get()); } } diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java index 4d0af5f85..d89445522 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java @@ -51,17 +51,43 @@ private TestMedia() { } /** - * Writes one second of 8 kHz G.711 mu-law in a WAV file. The WAV demuxer - * is part of the FFmpeg build and knows the format, but its decoder is - * not, so this opens as a reader and fails as a player. + * Writes one second of 8 kHz mono G.711 in a WAV file, the audio most IP + * cameras send. Every sample is the same quiet tone, not silence, which + * decodes to a constant non-zero value: 264 in A-law, 396 in mu-law. * * @param directory Where to write the file. + * @param aLaw A-law if true, mu-law otherwise. * * @return The file written. */ - static Path muLawWav(Path directory) throws IOException { + static Path g711Wav(Path directory, boolean aLaw) throws IOException { + return monoWav(directory, aLaw ? "alaw.wav" : "mulaw.wav", + aLaw ? 6 : 7, // WAVE_FORMAT_ALAW, WAVE_FORMAT_MULAW + 8, (byte) (aLaw ? 0xC5 : 0xDF)); + } + + /** + * Writes one second of 8 kHz 24-bit PCM in a WAV file. The WAV demuxer is + * part of the FFmpeg build and knows the format, but its decoder is not, + * so this opens as a reader and fails as a player. + * + * @param directory Where to write the file. + * + * @return The file written. + */ + static Path pcm24Wav(Path directory) throws IOException { + return monoWav(directory, "pcm24.wav", 1, 24, (byte) 0); // WAVE_FORMAT_PCM + } + + /** + * Writes one second of 8 kHz mono audio in a WAV file, each byte of its + * data the same. + */ + private static Path monoWav(Path directory, String name, int formatTag, + int bitsPerSample, byte fill) throws IOException { final int rate = 8000; - final int length = rate; + final int blockAlign = bitsPerSample / 8; + final int length = rate * blockAlign; ByteBuffer buffer = ByteBuffer.allocate(44 + length) .order(ByteOrder.LITTLE_ENDIAN); @@ -69,21 +95,20 @@ static Path muLawWav(Path directory) throws IOException { buffer.put(ascii("RIFF")).putInt(36 + length).put(ascii("WAVE")); buffer.put(ascii("fmt ")).putInt(16) - .putShort((short) 7) // WAVE_FORMAT_MULAW - .putShort((short) 1) // Channels - .putInt(rate) // Sample rate - .putInt(rate) // Byte rate - .putShort((short) 1) // Block align - .putShort((short) 8); // Bits per sample + .putShort((short) formatTag) + .putShort((short) 1) // Channels + .putInt(rate) // Sample rate + .putInt(rate * blockAlign) // Byte rate + .putShort((short) blockAlign) + .putShort((short) bitsPerSample); buffer.put(ascii("data")).putInt(length); while (buffer.hasRemaining()) { - // Silence in mu-law. - buffer.put((byte) 0xFF); + buffer.put(fill); } - return Files.write(directory.resolve("mulaw.wav"), buffer.array()); + return Files.write(directory.resolve(name), buffer.array()); } /**