Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/guide/media/media-files.md
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ The FFmpeg build is deliberately small, and carries only what this module plays:
| --- | --- |
| **Containers** | MP4 and MOV, Matroska and WebM, AVI, MPEG-TS, FLV, WAV, MP3, Ogg, FLAC, AAC |
| **Video** | H.264, H.265/HEVC (hardware only), VP8, VP9, AV1, MPEG-4 (including Xvid and DivX), Microsoft MPEG-4 v1 to v3, MJPEG |
| **Audio** | AAC, MP3, MP2, AC-3, Opus, Vorbis, FLAC, PCM, MS and IMA ADPCM |
| **Audio** | AAC, MP3, MP2, AC-3, Opus, Vorbis, FLAC, PCM, G.711 (A-law and µ-law), MS and IMA ADPCM |

Audio of any rate or layout is resampled to what WebRTC takes: 48 kHz 16-bit PCM, in mono or stereo. Video that decodes to I420, which is almost all 8-bit H.264, VP8, VP9 and MPEG-4, reaches the encoder without being copied. Anything else is converted first.

Expand Down
3 changes: 3 additions & 0 deletions webrtc-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ set(FFMPEG_COMPONENTS
--enable-decoder=pcm_s16be
--enable-decoder=pcm_u8
--enable-decoder=pcm_f32le
# G.711, the audio most IP cameras send over RTSP.
--enable-decoder=pcm_alaw
--enable-decoder=pcm_mulaw
# Audio common in AVI files and in MPEG transport streams.
--enable-decoder=mp2
--enable-decoder=ac3
Expand Down
6 changes: 6 additions & 0 deletions webrtc-media/src/main/cpp/include/media/MediaPlayer.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ namespace ffmpeg
// Returns 0 or a negative AVERROR.
int Initialize();

// Why Initialize failed, in words for an exception: which
// decoder is missing, where that is the reason. Empty if it has
// not failed.
const std::string & GetInitializeError() const;

void Play();
void Pause();

Expand Down Expand Up @@ -216,6 +221,7 @@ namespace ffmpeg
bool hardware_decoding_ = false;
bool has_video_ = false;
bool has_audio_ = false;
std::string initialize_error_;

std::thread thread_;
std::thread event_thread_;
Expand Down
8 changes: 8 additions & 0 deletions webrtc-media/src/main/cpp/include/media/VideoDecoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,14 @@ namespace ffmpeg
// Returns 0 or a negative AVERROR.
int Open(const AVStream * stream, bool hardware = false, int64_t start_time_us = 0);

// Whether the codec is decoded on the hardware only, never in
// software: HEVC. FFmpeg's decoder of it is in the build all the
// same, because the hardware decoders run inside it (it reads the
// headers and hands the slices to the hardware), but it decodes no
// picture itself. Where the hardware cannot, opening or decoding
// the stream fails.
static bool IsHardwareOnly(AVCodecID id);

void Close();

// Drops everything the decoder holds, which is what a seek needs:
Expand Down
8 changes: 7 additions & 1 deletion webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,16 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_create
int result = player->Initialize();

if (result < 0) {
std::string reason = player->GetInitializeError();

if (reason.empty()) {
reason = ffmpeg::ErrorText(result);
}

// Released before throwing, see DiscardPlayer.
player.reset();

ffmpeg::ThrowIOException(env, "Opening the decoders failed: " + ffmpeg::ErrorText(result));
ffmpeg::ThrowIOException(env, "Opening the decoders failed: " + reason);

return 0;
}
Expand Down
35 changes: 35 additions & 0 deletions webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,35 @@
#include <utility>

extern "C" {
#include <libavutil/avutil.h>
#include <libavutil/error.h>
}

namespace
{
// What an exception says when a decoder for the stream could not be
// opened. FFmpeg's own words for a missing decoder do not say which
// one, and with several streams the codec is what tells them apart.
std::string DescribeOpenError(const AVStream * stream, int error)
{
if (error != AVERROR_DECODER_NOT_FOUND) {
return ffmpeg::ErrorText(error);
}

const AVCodecID id = stream->codecpar->codec_id;
const std::string name = avcodec_get_name(id);

if (ffmpeg::VideoDecoder::IsHardwareOnly(id)) {
return "No hardware decoder for " + name + ", which is decoded in hardware only";
}

const char * type = av_get_media_type_string(stream->codecpar->codec_type);

return std::string("No decoder for ") + (type != nullptr ? type : "unknown")
+ " codec " + name;
}
}

namespace ffmpeg
{
MediaPlayer::MediaPlayer(std::unique_ptr<MediaReader> reader,
Expand Down Expand Up @@ -74,6 +100,8 @@ namespace ffmpeg
reader_->GetStartTimeUs());

if (result < 0) {
initialize_error_ = DescribeOpenError(reader_->GetVideoStream(), result);

return result;
}

Expand All @@ -84,6 +112,8 @@ namespace ffmpeg
int result = audio_decoder_.Open(reader_->GetAudioStream(), reader_->GetStartTimeUs());

if (result < 0) {
initialize_error_ = DescribeOpenError(reader_->GetAudioStream(), result);

return result;
}

Expand Down Expand Up @@ -113,6 +143,11 @@ namespace ffmpeg
return 0;
}

const std::string & MediaPlayer::GetInitializeError() const
{
return initialize_error_;
}

void MediaPlayer::Play()
{
std::lock_guard<std::mutex> lock(mutex_);
Expand Down
15 changes: 5 additions & 10 deletions webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,6 @@ namespace
#endif
}

// Whether the codec is decoded on the hardware only, never in software:
// HEVC. FFmpeg's decoder of it is in the build all the same, because the
// hardware decoders run inside it (it reads the headers and hands the
// slices to the hardware), but it decodes no picture itself. Where the
// hardware cannot, opening or decoding the stream fails.
bool IsHardwareOnly(AVCodecID id)
{
return id == AV_CODEC_ID_HEVC;
}

// The decoder to open for a codec. Software gets the one FFmpeg prefers.
// Hardware gets it too, unless it has no hardware configuration: where
// another decoder of the codec stands in front of the one that decodes on
Expand Down Expand Up @@ -114,6 +104,11 @@ namespace ffmpeg
Close();
}

bool VideoDecoder::IsHardwareOnly(AVCodecID id)
{
return id == AV_CODEC_ID_HEVC;
}

int VideoDecoder::Open(const AVStream * stream, bool hardware, int64_t start_time_us)
{
Close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ void hevcWithoutHardwareDoesNotOpen() throws Exception {
IOException error = assertThrows(IOException.class,
() -> new MediaPlayer(new MediaReader(path(HEVC)), source, null, hardware));

assertTrue(error.getMessage().contains("Decoder not found"), error.getMessage());
assertTrue(error.getMessage().endsWith("No hardware decoder for hevc, which is decoded in hardware only"),
error.getMessage());
}
finally {
source.dispose();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
Expand Down Expand Up @@ -1347,14 +1348,45 @@ void outlivesDisposedSources() throws Exception {

@Test
void failsWithoutDecoder() throws Exception {
Path file = TestMedia.muLawWav(tempDir);
Path file = TestMedia.pcm24Wav(tempDir);

try (Sources sources = new Sources()) {
// The reader opens, since the demuxer knows the format.
MediaReader reader = new MediaReader(file);

assertThrows(IOException.class,
IOException error = assertThrows(IOException.class,
() -> new MediaPlayer(reader, null, sources.audio));

// Names the codec, which the source may have several of.
assertEquals("Opening the decoders failed: No decoder for audio codec pcm_s24le",
error.getMessage());
}
}

@Test
void playsALaw() throws Exception {
assertPlaysG711(true);
}

@Test
void playsMuLaw() throws Exception {
assertPlaysG711(false);
}

private void assertPlaysG711(boolean aLaw) throws Exception {
// One second of 8 kHz mono, as an IP camera sends it.
Path file = TestMedia.g711Wav(tempDir, aLaw);

try (Playback playback = new Playback(new MediaReader(file))) {
playback.player.play();

assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream");
assertNull(playback.error.get());

assertEquals(48000, playback.sampleRate.get());
assertEquals(1, playback.channels.get());
assertEquals(100, playback.chunks.get());
assertEquals(48000, playback.nonZeroSamples.get());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,39 +51,64 @@ private TestMedia() {
}

/**
* Writes one second of 8 kHz G.711 mu-law in a WAV file. The WAV demuxer
* is part of the FFmpeg build and knows the format, but its decoder is
* not, so this opens as a reader and fails as a player.
* Writes one second of 8 kHz mono G.711 in a WAV file, the audio most IP
* cameras send. Every sample is the same quiet tone, not silence, which
* decodes to a constant non-zero value: 264 in A-law, 396 in mu-law.
*
* @param directory Where to write the file.
* @param aLaw A-law if true, mu-law otherwise.
*
* @return The file written.
*/
static Path muLawWav(Path directory) throws IOException {
static Path g711Wav(Path directory, boolean aLaw) throws IOException {
return monoWav(directory, aLaw ? "alaw.wav" : "mulaw.wav",
aLaw ? 6 : 7, // WAVE_FORMAT_ALAW, WAVE_FORMAT_MULAW
8, (byte) (aLaw ? 0xC5 : 0xDF));
}

/**
* Writes one second of 8 kHz 24-bit PCM in a WAV file. The WAV demuxer is
* part of the FFmpeg build and knows the format, but its decoder is not,
* so this opens as a reader and fails as a player.
*
* @param directory Where to write the file.
*
* @return The file written.
*/
static Path pcm24Wav(Path directory) throws IOException {
return monoWav(directory, "pcm24.wav", 1, 24, (byte) 0); // WAVE_FORMAT_PCM
}

/**
* Writes one second of 8 kHz mono audio in a WAV file, each byte of its
* data the same.
*/
private static Path monoWav(Path directory, String name, int formatTag,
int bitsPerSample, byte fill) throws IOException {
final int rate = 8000;
final int length = rate;
final int blockAlign = bitsPerSample / 8;
final int length = rate * blockAlign;

ByteBuffer buffer = ByteBuffer.allocate(44 + length)
.order(ByteOrder.LITTLE_ENDIAN);

buffer.put(ascii("RIFF")).putInt(36 + length).put(ascii("WAVE"));

buffer.put(ascii("fmt ")).putInt(16)
.putShort((short) 7) // WAVE_FORMAT_MULAW
.putShort((short) 1) // Channels
.putInt(rate) // Sample rate
.putInt(rate) // Byte rate
.putShort((short) 1) // Block align
.putShort((short) 8); // Bits per sample
.putShort((short) formatTag)
.putShort((short) 1) // Channels
.putInt(rate) // Sample rate
.putInt(rate * blockAlign) // Byte rate
.putShort((short) blockAlign)
.putShort((short) bitsPerSample);

buffer.put(ascii("data")).putInt(length);

while (buffer.hasRemaining()) {
// Silence in mu-law.
buffer.put((byte) 0xFF);
buffer.put(fill);
}

return Files.write(directory.resolve("mulaw.wav"), buffer.array());
return Files.write(directory.resolve(name), buffer.array());
}

/**
Expand Down
Loading