/* * Copyright 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ package org.webrtc; import android.media.MediaCodec; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.os.SystemClock; import android.view.Surface; import androidx.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.BlockingDeque; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import org.webrtc.ThreadUtils.ThreadChecker; /** * Android hardware video decoder. */ class AndroidVideoDecoder implements VideoDecoder, VideoSink { private static final String TAG = "AndroidVideoDecoder"; // MediaCodec.release() occasionally hangs. Release stops waiting and reports failure after // this timeout. private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // WebRTC queues input frames quickly in the beginning on the call. Wait for input buffers with a // long timeout (500 ms) to prevent this from causing the codec to return an error. private static final int DEQUEUE_INPUT_TIMEOUT_US = 500000; // Dequeuing an output buffer will block until a buffer is available (up to 100 milliseconds). // If this timeout is exceeded, the output thread will unblock and check if the decoder is still // running. If it is, it will block on dequeue again. Otherwise, it will stop and release the // MediaCodec. private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; private final MediaCodecWrapperFactory mediaCodecWrapperFactory; private final String codecName; private final VideoCodecMimeType codecType; private static class FrameInfo { final long decodeStartTimeMs; final int rotation; FrameInfo(long decodeStartTimeMs, int rotation) { this.decodeStartTimeMs = decodeStartTimeMs; this.rotation = rotation; } } private final BlockingDeque frameInfos; private int colorFormat; // Output thread runs a loop which polls MediaCodec for decoded output buffers. It reformats // those buffers into VideoFrames and delivers them to the callback. Variable is set on decoder // thread and is immutable while the codec is running. @Nullable private Thread outputThread; // Checker that ensures work is run on the output thread. private ThreadChecker outputThreadChecker; // Checker that ensures work is run on the decoder thread. The decoder thread is owned by the // caller and must be used to call initDecode, decode, and release. private ThreadChecker decoderThreadChecker; private volatile boolean running; @Nullable private volatile Exception shutdownException; // Dimensions (width, height, stride, and sliceHeight) may be accessed by either the decode thread // or the output thread. Accesses should be protected with this lock. private final Object dimensionLock = new Object(); private int width; private int height; private int stride; private int sliceHeight; // Whether the decoder has finished the first frame. The codec may not change output dimensions // after delivering the first frame. Only accessed on the output thread while the decoder is // running. private boolean hasDecodedFirstFrame; // Whether the decoder has seen a key frame. The first frame must be a key frame. Only accessed // on the decoder thread. private boolean keyFrameRequired; private final @Nullable EglBase.Context sharedContext; // Valid and immutable while the decoder is running. @Nullable private SurfaceTextureHelper surfaceTextureHelper; @Nullable private Surface surface; private static class DecodedTextureMetadata { final long presentationTimestampUs; final Integer decodeTimeMs; DecodedTextureMetadata(long presentationTimestampUs, Integer decodeTimeMs) { this.presentationTimestampUs = presentationTimestampUs; this.decodeTimeMs = decodeTimeMs; } } // Metadata for the last frame rendered to the texture. private final Object renderedTextureMetadataLock = new Object(); @Nullable private DecodedTextureMetadata renderedTextureMetadata; // Decoding proceeds asynchronously. This callback returns decoded frames to the caller. Valid // and immutable while the decoder is running. @Nullable private Callback callback; // Valid and immutable while the decoder is running. @Nullable private MediaCodecWrapper codec; AndroidVideoDecoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, VideoCodecMimeType codecType, int colorFormat, @Nullable EglBase.Context sharedContext) { if (!isSupportedColorFormat(colorFormat)) { throw new IllegalArgumentException("Unsupported color format: " + colorFormat); } Logging.d(TAG, "ctor name: " + codecName + " type: " + codecType + " color format: " + colorFormat + " context: " + sharedContext); this.mediaCodecWrapperFactory = mediaCodecWrapperFactory; this.codecName = codecName; this.codecType = codecType; this.colorFormat = colorFormat; this.sharedContext = sharedContext; this.frameInfos = new LinkedBlockingDeque<>(); } @Override public VideoCodecStatus initDecode(Settings settings, Callback callback) { this.decoderThreadChecker = new ThreadChecker(); this.callback = callback; if (sharedContext != null) { surfaceTextureHelper = createSurfaceTextureHelper(); surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); surfaceTextureHelper.startListening(this); } return initDecodeInternal(settings.width, settings.height); } // Internal variant is used when restarting the codec due to reconfiguration. private VideoCodecStatus initDecodeInternal(int width, int height) { decoderThreadChecker.checkIsOnValidThread(); Logging.d(TAG, "initDecodeInternal name: " + codecName + " type: " + codecType + " width: " + width + " height: " + height + " color format: " + colorFormat); if (outputThread != null) { Logging.e(TAG, "initDecodeInternal called while the codec is already running"); return VideoCodecStatus.FALLBACK_SOFTWARE; } // Note: it is not necessary to initialize dimensions under the lock, since the output thread // is not running. this.width = width; this.height = height; stride = width; sliceHeight = height; hasDecodedFirstFrame = false; keyFrameRequired = true; try { codec = mediaCodecWrapperFactory.createByCodecName(codecName); } catch (IOException | IllegalArgumentException | IllegalStateException e) { Logging.e(TAG, "Cannot create media decoder " + codecName); return VideoCodecStatus.FALLBACK_SOFTWARE; } try { MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height); if (sharedContext == null) { format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); } codec.configure(format, surface, null, 0); codec.start(); } catch (IllegalStateException | IllegalArgumentException e) { Logging.e(TAG, "initDecode failed", e); release(); return VideoCodecStatus.FALLBACK_SOFTWARE; } running = true; outputThread = createOutputThread(); outputThread.start(); Logging.d(TAG, "initDecodeInternal done"); return VideoCodecStatus.OK; } @Override public VideoCodecStatus decode(EncodedImage frame, DecodeInfo info) { decoderThreadChecker.checkIsOnValidThread(); if (codec == null || callback == null) { Logging.d(TAG, "decode uninitalized, codec: " + (codec != null) + ", callback: " + callback); return VideoCodecStatus.UNINITIALIZED; } if (frame.buffer == null) { Logging.e(TAG, "decode() - no input data"); return VideoCodecStatus.ERR_PARAMETER; } int size = frame.buffer.remaining(); if (size == 0) { Logging.e(TAG, "decode() - input buffer empty"); return VideoCodecStatus.ERR_PARAMETER; } // Load dimensions from shared memory under the dimension lock. final int width; final int height; synchronized (dimensionLock) { width = this.width; height = this.height; } // Check if the resolution changed and reset the codec if necessary. if (frame.encodedWidth * frame.encodedHeight > 0 && (frame.encodedWidth != width || frame.encodedHeight != height)) { VideoCodecStatus status = reinitDecode(frame.encodedWidth, frame.encodedHeight); if (status != VideoCodecStatus.OK) { return status; } } if (keyFrameRequired) { // Need to process a key frame first. if (frame.frameType != EncodedImage.FrameType.VideoFrameKey) { Logging.e(TAG, "decode() - key frame required first"); return VideoCodecStatus.NO_OUTPUT; } } int index; try { index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US); } catch (IllegalStateException e) { Logging.e(TAG, "dequeueInputBuffer failed", e); return VideoCodecStatus.ERROR; } if (index < 0) { // Decoder is falling behind. No input buffers available. // The decoder can't simply drop frames; it might lose a key frame. Logging.e(TAG, "decode() - no HW buffers available; decoder falling behind"); return VideoCodecStatus.ERROR; } ByteBuffer buffer; try { buffer = codec.getInputBuffer(index); } catch (IllegalStateException e) { Logging.e(TAG, "getInputBuffer with index=" + index + " failed", e); return VideoCodecStatus.ERROR; } if (buffer.capacity() < size) { Logging.e(TAG, "decode() - HW buffer too small"); return VideoCodecStatus.ERROR; } buffer.put(frame.buffer); frameInfos.offer(new FrameInfo(SystemClock.elapsedRealtime(), frame.rotation)); try { codec.queueInputBuffer(index, 0 /* offset */, size, TimeUnit.NANOSECONDS.toMicros(frame.captureTimeNs), 0 /* flags */); } catch (IllegalStateException e) { Logging.e(TAG, "queueInputBuffer failed", e); frameInfos.pollLast(); return VideoCodecStatus.ERROR; } if (keyFrameRequired) { keyFrameRequired = false; } return VideoCodecStatus.OK; } @Override public String getImplementationName() { return codecName; } @Override public VideoCodecStatus release() { // TODO(sakal): This is not called on the correct thread but is still called synchronously. // Re-enable the check once this is called on the correct thread. // decoderThreadChecker.checkIsOnValidThread(); Logging.d(TAG, "release"); VideoCodecStatus status = releaseInternal(); if (surface != null) { releaseSurface(); surface = null; surfaceTextureHelper.stopListening(); surfaceTextureHelper.dispose(); surfaceTextureHelper = null; } synchronized (renderedTextureMetadataLock) { renderedTextureMetadata = null; } callback = null; frameInfos.clear(); return status; } // Internal variant is used when restarting the codec due to reconfiguration. private VideoCodecStatus releaseInternal() { if (!running) { Logging.d(TAG, "release: Decoder is not running."); return VideoCodecStatus.OK; } try { // The outputThread actually stops and releases the codec once running is false. running = false; if (!ThreadUtils.joinUninterruptibly(outputThread, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { // Log an exception to capture the stack trace and turn it into a TIMEOUT error. Logging.e(TAG, "Media decoder release timeout", new RuntimeException()); return VideoCodecStatus.TIMEOUT; } if (shutdownException != null) { // Log the exception and turn it into an error. Wrap the exception in a new exception to // capture both the output thread's stack trace and this thread's stack trace. Logging.e(TAG, "Media decoder release error", new RuntimeException(shutdownException)); shutdownException = null; return VideoCodecStatus.ERROR; } } finally { codec = null; outputThread = null; } return VideoCodecStatus.OK; } private VideoCodecStatus reinitDecode(int newWidth, int newHeight) { decoderThreadChecker.checkIsOnValidThread(); VideoCodecStatus status = releaseInternal(); if (status != VideoCodecStatus.OK) { return status; } return initDecodeInternal(newWidth, newHeight); } private Thread createOutputThread() { return new Thread("AndroidVideoDecoder.outputThread") { @Override public void run() { outputThreadChecker = new ThreadChecker(); while (running) { deliverDecodedFrame(); } releaseCodecOnOutputThread(); } }; } // Visible for testing. protected void deliverDecodedFrame() { outputThreadChecker.checkIsOnValidThread(); try { MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); // Block until an output buffer is available (up to 100 milliseconds). If the timeout is // exceeded, deliverDecodedFrame() will be called again on the next iteration of the output // thread's loop. Blocking here prevents the output thread from busy-waiting while the codec // is idle. int index = codec.dequeueOutputBuffer(info, DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US); if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { reformat(codec.getOutputFormat()); return; } if (index < 0) { Logging.v(TAG, "dequeueOutputBuffer returned " + index); return; } FrameInfo frameInfo = frameInfos.poll(); Integer decodeTimeMs = null; int rotation = 0; if (frameInfo != null) { decodeTimeMs = (int) (SystemClock.elapsedRealtime() - frameInfo.decodeStartTimeMs); rotation = frameInfo.rotation; } hasDecodedFirstFrame = true; if (surfaceTextureHelper != null) { deliverTextureFrame(index, info, rotation, decodeTimeMs); } else { deliverByteFrame(index, info, rotation, decodeTimeMs); } } catch (IllegalStateException e) { Logging.e(TAG, "deliverDecodedFrame failed", e); } } private void deliverTextureFrame(final int index, final MediaCodec.BufferInfo info, final int rotation, final Integer decodeTimeMs) { // Load dimensions from shared memory under the dimension lock. final int width; final int height; synchronized (dimensionLock) { width = this.width; height = this.height; } synchronized (renderedTextureMetadataLock) { if (renderedTextureMetadata != null) { codec.releaseOutputBuffer(index, false); return; // We are still waiting for texture for the previous frame, drop this one. } surfaceTextureHelper.setTextureSize(width, height); surfaceTextureHelper.setFrameRotation(rotation); renderedTextureMetadata = new DecodedTextureMetadata(info.presentationTimeUs, decodeTimeMs); codec.releaseOutputBuffer(index, /* render= */ true); } } @Override public void onFrame(VideoFrame frame) { final VideoFrame newFrame; final Integer decodeTimeMs; final long timestampNs; synchronized (renderedTextureMetadataLock) { if (renderedTextureMetadata == null) { throw new IllegalStateException( "Rendered texture metadata was null in onTextureFrameAvailable."); } timestampNs = renderedTextureMetadata.presentationTimestampUs * 1000; decodeTimeMs = renderedTextureMetadata.decodeTimeMs; renderedTextureMetadata = null; } // Change timestamp of frame. final VideoFrame frameWithModifiedTimeStamp = new VideoFrame(frame.getBuffer(), frame.getRotation(), timestampNs); callback.onDecodedFrame(frameWithModifiedTimeStamp, decodeTimeMs, null /* qp */); } private void deliverByteFrame( int index, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) { // Load dimensions from shared memory under the dimension lock. int width; int height; int stride; int sliceHeight; synchronized (dimensionLock) { width = this.width; height = this.height; stride = this.stride; sliceHeight = this.sliceHeight; } // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2) // bytes for each of the U and V channels. if (info.size < width * height * 3 / 2) { Logging.e(TAG, "Insufficient output buffer size: " + info.size); return; } if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) { // Some codecs (Exynos) report an incorrect stride. Correct it here. // Expected size == stride * height * 3 / 2. A bit of algebra gives the correct stride as // 2 * size / (3 * height). stride = info.size * 2 / (height * 3); } ByteBuffer buffer = codec.getOutputBuffer(index); buffer.position(info.offset); buffer.limit(info.offset + info.size); buffer = buffer.slice(); final VideoFrame.Buffer frameBuffer; if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) { frameBuffer = copyI420Buffer(buffer, stride, sliceHeight, width, height); } else { // All other supported color formats are NV12. frameBuffer = copyNV12ToI420Buffer(buffer, stride, sliceHeight, width, height); } codec.releaseOutputBuffer(index, /* render= */ false); long presentationTimeNs = info.presentationTimeUs * 1000; VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs); // Note that qp is parsed on the C++ side. callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */); frame.release(); } private VideoFrame.Buffer copyNV12ToI420Buffer( ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { // toI420 copies the buffer. return new NV12Buffer(width, height, stride, sliceHeight, buffer, null /* releaseCallback */) .toI420(); } private VideoFrame.Buffer copyI420Buffer( ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { if (stride % 2 != 0) { throw new AssertionError("Stride is not divisible by two: " + stride); } // Note that the case with odd `sliceHeight` is handled in a special way. // The chroma height contained in the payload is rounded down instead of // up, making it one row less than what we expect in WebRTC. Therefore, we // have to duplicate the last chroma rows for this case. Also, the offset // between the Y plane and the U plane is unintuitive for this case. See // http://bugs.webrtc.org/6651 for more info. final int chromaWidth = (width + 1) / 2; final int chromaHeight = (sliceHeight % 2 == 0) ? (height + 1) / 2 : height / 2; final int uvStride = stride / 2; final int yPos = 0; final int yEnd = yPos + stride * height; final int uPos = yPos + stride * sliceHeight; final int uEnd = uPos + uvStride * chromaHeight; final int vPos = uPos + uvStride * sliceHeight / 2; final int vEnd = vPos + uvStride * chromaHeight; VideoFrame.I420Buffer frameBuffer = allocateI420Buffer(width, height); buffer.limit(yEnd); buffer.position(yPos); copyPlane( buffer.slice(), stride, frameBuffer.getDataY(), frameBuffer.getStrideY(), width, height); buffer.limit(uEnd); buffer.position(uPos); copyPlane(buffer.slice(), uvStride, frameBuffer.getDataU(), frameBuffer.getStrideU(), chromaWidth, chromaHeight); if (sliceHeight % 2 == 1) { buffer.position(uPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. ByteBuffer dataU = frameBuffer.getDataU(); dataU.position(frameBuffer.getStrideU() * chromaHeight); // Seek to beginning of last row. dataU.put(buffer); // Copy the last row. } buffer.limit(vEnd); buffer.position(vPos); copyPlane(buffer.slice(), uvStride, frameBuffer.getDataV(), frameBuffer.getStrideV(), chromaWidth, chromaHeight); if (sliceHeight % 2 == 1) { buffer.position(vPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. ByteBuffer dataV = frameBuffer.getDataV(); dataV.position(frameBuffer.getStrideV() * chromaHeight); // Seek to beginning of last row. dataV.put(buffer); // Copy the last row. } return frameBuffer; } private void reformat(MediaFormat format) { outputThreadChecker.checkIsOnValidThread(); Logging.d(TAG, "Decoder format changed: " + format); final int newWidth; final int newHeight; if (format.containsKey(MediaFormat.KEY_CROP_LEFT) && format.containsKey(MediaFormat.KEY_CROP_RIGHT) && format.containsKey(MediaFormat.KEY_CROP_BOTTOM) && format.containsKey(MediaFormat.KEY_CROP_TOP)) { newWidth = 1 + format.getInteger(MediaFormat.KEY_CROP_RIGHT) - format.getInteger(MediaFormat.KEY_CROP_LEFT); newHeight = 1 + format.getInteger(MediaFormat.KEY_CROP_BOTTOM) - format.getInteger(MediaFormat.KEY_CROP_TOP); } else { newWidth = format.getInteger(MediaFormat.KEY_WIDTH); newHeight = format.getInteger(MediaFormat.KEY_HEIGHT); } // Compare to existing width, height, and save values under the dimension lock. synchronized (dimensionLock) { if (newWidth != width || newHeight != height) { if (hasDecodedFirstFrame) { stopOnOutputThread(new RuntimeException("Unexpected size change. " + "Configured " + width + "*" + height + ". " + "New " + newWidth + "*" + newHeight)); return; } else if (newWidth <= 0 || newHeight <= 0) { Logging.w(TAG, "Unexpected format dimensions. Configured " + width + "*" + height + ". " + "New " + newWidth + "*" + newHeight + ". Skip it"); return; } width = newWidth; height = newHeight; } } // Note: texture mode ignores colorFormat. Hence, if the texture helper is non-null, skip // color format updates. if (surfaceTextureHelper == null && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); if (!isSupportedColorFormat(colorFormat)) { stopOnOutputThread(new IllegalStateException("Unsupported color format: " + colorFormat)); return; } } // Save stride and sliceHeight under the dimension lock. synchronized (dimensionLock) { if (format.containsKey(MediaFormat.KEY_STRIDE)) { stride = format.getInteger(MediaFormat.KEY_STRIDE); } if (format.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { sliceHeight = format.getInteger(MediaFormat.KEY_SLICE_HEIGHT); } Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); stride = Math.max(width, stride); sliceHeight = Math.max(height, sliceHeight); } } private void releaseCodecOnOutputThread() { outputThreadChecker.checkIsOnValidThread(); Logging.d(TAG, "Releasing MediaCodec on output thread"); try { codec.stop(); } catch (Exception e) { Logging.e(TAG, "Media decoder stop failed", e); } try { codec.release(); } catch (Exception e) { Logging.e(TAG, "Media decoder release failed", e); // Propagate exceptions caught during release back to the main thread. shutdownException = e; } Logging.d(TAG, "Release on output thread done"); } private void stopOnOutputThread(Exception e) { outputThreadChecker.checkIsOnValidThread(); running = false; shutdownException = e; } private boolean isSupportedColorFormat(int colorFormat) { for (int supported : MediaCodecUtils.DECODER_COLOR_FORMATS) { if (supported == colorFormat) { return true; } } return false; } // Visible for testing. protected SurfaceTextureHelper createSurfaceTextureHelper() { return SurfaceTextureHelper.create("decoder-texture-thread", sharedContext); } // Visible for testing. // TODO(sakal): Remove once Robolectric commit fa991a0 has been rolled to WebRTC. protected void releaseSurface() { surface.release(); } // Visible for testing. protected VideoFrame.I420Buffer allocateI420Buffer(int width, int height) { return JavaI420Buffer.allocate(width, height); } // Visible for testing. protected void copyPlane( ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) { YuvHelper.copyPlane(src, srcStride, dst, dstStride, width, height); } }