/* * Copyright (C) 2016 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec; import android.annotation.TargetApi; import android.media.MediaCodec; import android.media.MediaCodec.CodecException; import android.media.MediaCodec.CryptoException; import android.media.MediaCrypto; import android.media.MediaCryptoException; import android.media.MediaFormat; import android.os.Bundle; import android.os.SystemClock; import androidx.annotation.CheckResult; import androidx.annotation.IntDef; import androidx.annotation.Nullable; import org.mozilla.thirdparty.com.google.android.exoplayer2.BaseRenderer; import org.mozilla.thirdparty.com.google.android.exoplayer2.C; import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; import org.mozilla.thirdparty.com.google.android.exoplayer2.FormatHolder; import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderCounters; import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; import org.mozilla.thirdparty.com.google.android.exoplayer2.drm.DrmSession; import org.mozilla.thirdparty.com.google.android.exoplayer2.drm.DrmSession.DrmSessionException; import org.mozilla.thirdparty.com.google.android.exoplayer2.drm.DrmSessionManager; import org.mozilla.thirdparty.com.google.android.exoplayer2.drm.FrameworkMediaCrypto; import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryException; import org.mozilla.thirdparty.com.google.android.exoplayer2.source.MediaPeriod; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Assertions; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Log; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.NalUnitUtil; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.TimedValueQueue; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.TraceUtil; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Util; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; /** * An abstract renderer that uses {@link MediaCodec} to decode samples for rendering. */ public abstract class MediaCodecRenderer extends BaseRenderer { /** Thrown when a failure occurs instantiating a decoder. */ public static class DecoderInitializationException extends Exception { private static final int CUSTOM_ERROR_CODE_BASE = -50000; private static final int NO_SUITABLE_DECODER_ERROR = CUSTOM_ERROR_CODE_BASE + 1; private static final int DECODER_QUERY_ERROR = CUSTOM_ERROR_CODE_BASE + 2; /** * The mime type for which a decoder was being initialized. */ public final String mimeType; /** * Whether it was required that the decoder support a secure output path. */ public final boolean secureDecoderRequired; /** * The {@link MediaCodecInfo} of the decoder that failed to initialize. Null if no suitable * decoder was found. */ @Nullable public final MediaCodecInfo codecInfo; /** An optional developer-readable diagnostic information string. May be null. */ @Nullable public final String diagnosticInfo; /** * If the decoder failed to initialize and another decoder being used as a fallback also failed * to initialize, the {@link DecoderInitializationException} for the fallback decoder. Null if * there was no fallback decoder or no suitable decoders were found. */ @Nullable public final DecoderInitializationException fallbackDecoderInitializationException; public DecoderInitializationException(Format format, Throwable cause, boolean secureDecoderRequired, int errorCode) { this( "Decoder init failed: [" + errorCode + "], " + format, cause, format.sampleMimeType, secureDecoderRequired, /* mediaCodecInfo= */ null, buildCustomDiagnosticInfo(errorCode), /* fallbackDecoderInitializationException= */ null); } public DecoderInitializationException( Format format, Throwable cause, boolean secureDecoderRequired, MediaCodecInfo mediaCodecInfo) { this( "Decoder init failed: " + mediaCodecInfo.name + ", " + format, cause, format.sampleMimeType, secureDecoderRequired, mediaCodecInfo, Util.SDK_INT >= 21 ? getDiagnosticInfoV21(cause) : null, /* fallbackDecoderInitializationException= */ null); } private DecoderInitializationException( String message, Throwable cause, String mimeType, boolean secureDecoderRequired, @Nullable MediaCodecInfo mediaCodecInfo, @Nullable String diagnosticInfo, @Nullable DecoderInitializationException fallbackDecoderInitializationException) { super(message, cause); this.mimeType = mimeType; this.secureDecoderRequired = secureDecoderRequired; this.codecInfo = mediaCodecInfo; this.diagnosticInfo = diagnosticInfo; this.fallbackDecoderInitializationException = fallbackDecoderInitializationException; } @CheckResult private DecoderInitializationException copyWithFallbackException( DecoderInitializationException fallbackException) { return new DecoderInitializationException( getMessage(), getCause(), mimeType, secureDecoderRequired, codecInfo, diagnosticInfo, fallbackException); } @TargetApi(21) private static String getDiagnosticInfoV21(Throwable cause) { if (cause instanceof CodecException) { return ((CodecException) cause).getDiagnosticInfo(); } return null; } private static String buildCustomDiagnosticInfo(int errorCode) { String sign = errorCode < 0 ? "neg_" : ""; return "com.google.android.exoplayer2.mediacodec.MediaCodecRenderer_" + sign + Math.abs(errorCode); } } /** Thrown when a failure occurs in the decoder. */ public static class DecoderException extends Exception { /** The {@link MediaCodecInfo} of the decoder that failed. Null if unknown. */ @Nullable public final MediaCodecInfo codecInfo; /** An optional developer-readable diagnostic information string. May be null. */ @Nullable public final String diagnosticInfo; public DecoderException(Throwable cause, @Nullable MediaCodecInfo codecInfo) { super("Decoder failed: " + (codecInfo == null ? null : codecInfo.name), cause); this.codecInfo = codecInfo; diagnosticInfo = Util.SDK_INT >= 21 ? getDiagnosticInfoV21(cause) : null; } @TargetApi(21) private static String getDiagnosticInfoV21(Throwable cause) { if (cause instanceof CodecException) { return ((CodecException) cause).getDiagnosticInfo(); } return null; } } /** Indicates no codec operating rate should be set. */ protected static final float CODEC_OPERATING_RATE_UNSET = -1; private static final String TAG = "MediaCodecRenderer"; /** * If the {@link MediaCodec} is hotswapped (i.e. replaced during playback), this is the period of * time during which {@link #isReady()} will report true regardless of whether the new codec has * output frames that are ready to be rendered. *
* This allows codec hotswapping to be performed seamlessly, without interrupting the playback of
* other renderers, provided the new codec is able to decode some frames within this time period.
*/
private static final long MAX_CODEC_HOTSWAP_TIME_MS = 1000;
/**
* The possible return values for {@link #canKeepCodec(MediaCodec, MediaCodecInfo, Format,
* Format)}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
KEEP_CODEC_RESULT_NO,
KEEP_CODEC_RESULT_YES_WITH_FLUSH,
KEEP_CODEC_RESULT_YES_WITH_RECONFIGURATION,
KEEP_CODEC_RESULT_YES_WITHOUT_RECONFIGURATION
})
protected @interface KeepCodecResult {}
/** The codec cannot be kept. */
protected static final int KEEP_CODEC_RESULT_NO = 0;
/** The codec can be kept, but must be flushed. */
protected static final int KEEP_CODEC_RESULT_YES_WITH_FLUSH = 1;
/**
* The codec can be kept. It does not need to be flushed, but must be reconfigured by prefixing
* the next input buffer with the new format's configuration data.
*/
protected static final int KEEP_CODEC_RESULT_YES_WITH_RECONFIGURATION = 2;
/** The codec can be kept. It does not need to be flushed and no reconfiguration is required. */
protected static final int KEEP_CODEC_RESULT_YES_WITHOUT_RECONFIGURATION = 3;
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
RECONFIGURATION_STATE_NONE,
RECONFIGURATION_STATE_WRITE_PENDING,
RECONFIGURATION_STATE_QUEUE_PENDING
})
private @interface ReconfigurationState {}
/**
* There is no pending adaptive reconfiguration work.
*/
private static final int RECONFIGURATION_STATE_NONE = 0;
/**
* Codec configuration data needs to be written into the next buffer.
*/
private static final int RECONFIGURATION_STATE_WRITE_PENDING = 1;
/**
* Codec configuration data has been written into the next buffer, but that buffer still needs to
* be returned to the codec.
*/
private static final int RECONFIGURATION_STATE_QUEUE_PENDING = 2;
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({DRAIN_STATE_NONE, DRAIN_STATE_SIGNAL_END_OF_STREAM, DRAIN_STATE_WAIT_END_OF_STREAM})
private @interface DrainState {}
/** The codec is not being drained. */
private static final int DRAIN_STATE_NONE = 0;
/** The codec needs to be drained, but we haven't signaled an end of stream to it yet. */
private static final int DRAIN_STATE_SIGNAL_END_OF_STREAM = 1;
/** The codec needs to be drained, and we're waiting for it to output an end of stream. */
private static final int DRAIN_STATE_WAIT_END_OF_STREAM = 2;
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
DRAIN_ACTION_NONE,
DRAIN_ACTION_FLUSH,
DRAIN_ACTION_UPDATE_DRM_SESSION,
DRAIN_ACTION_REINITIALIZE
})
private @interface DrainAction {}
/** No special action should be taken. */
private static final int DRAIN_ACTION_NONE = 0;
/** The codec should be flushed. */
private static final int DRAIN_ACTION_FLUSH = 1;
/** The codec should be flushed and updated to use the pending DRM session. */
private static final int DRAIN_ACTION_UPDATE_DRM_SESSION = 2;
/** The codec should be reinitialized. */
private static final int DRAIN_ACTION_REINITIALIZE = 3;
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
ADAPTATION_WORKAROUND_MODE_NEVER,
ADAPTATION_WORKAROUND_MODE_SAME_RESOLUTION,
ADAPTATION_WORKAROUND_MODE_ALWAYS
})
private @interface AdaptationWorkaroundMode {}
/**
* The adaptation workaround is never used.
*/
private static final int ADAPTATION_WORKAROUND_MODE_NEVER = 0;
/**
* The adaptation workaround is used when adapting between formats of the same resolution only.
*/
private static final int ADAPTATION_WORKAROUND_MODE_SAME_RESOLUTION = 1;
/**
* The adaptation workaround is always used when adapting between formats.
*/
private static final int ADAPTATION_WORKAROUND_MODE_ALWAYS = 2;
/**
* H.264/AVC buffer to queue when using the adaptation workaround (see {@link
* #codecAdaptationWorkaroundMode(String)}. Consists of three NAL units with start codes: Baseline
* sequence/picture parameter sets and a 32 * 32 pixel IDR slice. This stream can be queued to
* force a resolution change when adapting to a new format.
*/
private static final byte[] ADAPTATION_WORKAROUND_BUFFER =
new byte[] {
0, 0, 1, 103, 66, -64, 11, -38, 37, -112, 0, 0, 1, 104, -50, 15, 19, 32, 0, 0, 1, 101, -120,
-124, 13, -50, 113, 24, -96, 0, 47, -65, 28, 49, -61, 39, 93, 120
};
private static final int ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT = 32;
private final MediaCodecSelector mediaCodecSelector;
@Nullable private final DrmSessionManager This method is experimental, and will be renamed or removed in a future release. It should
* only be called before the renderer is used.
*
* @param renderTimeLimitMs The render time limit in milliseconds, or {@link C#TIME_UNSET} for no
* limit.
*/
public void experimental_setRenderTimeLimitMs(long renderTimeLimitMs) {
this.renderTimeLimitMs = renderTimeLimitMs;
}
/**
* Skip calling {@link MediaCodec#stop()} when the underlying MediaCodec is going to be released.
*
* By default, when the MediaCodecRenderer is releasing the underlying {@link MediaCodec}, it
* first calls {@link MediaCodec#stop()} and then calls {@link MediaCodec#release()}. If this
* feature is enabled, the MediaCodecRenderer will skip the call to {@link MediaCodec#stop()}.
*
* This method is experimental, and will be renamed or removed in a future release. It should
* only be called before the renderer is used.
*
* @param enabled enable or disable the feature.
*/
public void experimental_setSkipMediaCodecStopOnRelease(boolean enabled) {
skipMediaCodecStopOnRelease = enabled;
}
@Override
@AdaptiveSupport
public final int supportsMixedMimeTypeAdaptation() {
return ADAPTIVE_NOT_SEAMLESS;
}
@Override
@Capabilities
public final int supportsFormat(Format format) throws ExoPlaybackException {
try {
return supportsFormat(mediaCodecSelector, drmSessionManager, format);
} catch (DecoderQueryException e) {
throw createRendererException(e, format);
}
}
/**
* Returns the {@link Capabilities} for the given {@link Format}.
*
* @param mediaCodecSelector The decoder selector.
* @param drmSessionManager The renderer's {@link DrmSessionManager}.
* @param format The {@link Format}.
* @return The {@link Capabilities} for this {@link Format}.
* @throws DecoderQueryException If there was an error querying decoders.
*/
@Capabilities
protected abstract int supportsFormat(
MediaCodecSelector mediaCodecSelector,
@Nullable DrmSessionManager The implementation of this method calls {@link #flushOrReleaseCodec()}, and {@link
* #maybeInitCodec()} if the codec needs to be re-instantiated.
*
* @return Whether the codec was released and reinitialized, rather than being flushed.
* @throws ExoPlaybackException If an error occurs re-instantiating the codec.
*/
protected final boolean flushOrReinitializeCodec() throws ExoPlaybackException {
boolean released = flushOrReleaseCodec();
if (released) {
maybeInitCodec();
}
return released;
}
/**
* Flushes the codec. If flushing is not possible, the codec will be released. This method is a
* no-op if the codec is {@code null}.
*
* @return Whether the codec was released.
*/
protected boolean flushOrReleaseCodec() {
if (codec == null) {
return false;
}
if (codecDrainAction == DRAIN_ACTION_REINITIALIZE
|| codecNeedsFlushWorkaround
|| (codecNeedsSosFlushWorkaround && !codecHasOutputMediaFormat)
|| (codecNeedsEosFlushWorkaround && codecReceivedEos)) {
releaseCodec();
return true;
}
codec.flush();
resetInputBuffer();
resetOutputBuffer();
codecHotswapDeadlineMs = C.TIME_UNSET;
codecReceivedEos = false;
codecReceivedBuffers = false;
waitingForFirstSyncSample = true;
codecNeedsAdaptationWorkaroundBuffer = false;
shouldSkipAdaptationWorkaroundOutputBuffer = false;
isDecodeOnlyOutputBuffer = false;
isLastOutputBuffer = false;
waitingForKeys = false;
decodeOnlyPresentationTimestamps.clear();
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
codecDrainState = DRAIN_STATE_NONE;
codecDrainAction = DRAIN_ACTION_NONE;
// Reconfiguration data sent shortly before the flush may not have been processed by the
// decoder. If the codec has been reconfigured we always send reconfiguration data again to
// guarantee that it's processed.
codecReconfigurationState =
codecReconfigured ? RECONFIGURATION_STATE_WRITE_PENDING : RECONFIGURATION_STATE_NONE;
return false;
}
protected DecoderException createDecoderException(
Throwable cause, @Nullable MediaCodecInfo codecInfo) {
return new DecoderException(cause, codecInfo);
}
/** Reads into {@link #flagsOnlyBuffer} and returns whether a {@link Format} was read. */
private boolean readToFlagsOnlyBuffer(boolean requireFormat) throws ExoPlaybackException {
FormatHolder formatHolder = getFormatHolder();
flagsOnlyBuffer.clear();
int result = readSource(formatHolder, flagsOnlyBuffer, requireFormat);
if (result == C.RESULT_FORMAT_READ) {
onInputFormatChanged(formatHolder);
return true;
} else if (result == C.RESULT_BUFFER_READ && flagsOnlyBuffer.isEndOfStream()) {
inputStreamEnded = true;
processEndOfStream();
}
return false;
}
private void maybeInitCodecWithFallback(
MediaCrypto crypto, boolean mediaCryptoRequiresSecureDecoder)
throws DecoderInitializationException {
if (availableCodecInfos == null) {
try {
List
* The default implementation is a no-op.
*
* @param name The name of the codec that was initialized.
* @param initializedTimestampMs {@link SystemClock#elapsedRealtime()} when initialization
* finished.
* @param initializationDurationMs The time taken to initialize the codec in milliseconds.
*/
protected void onCodecInitialized(String name, long initializedTimestampMs,
long initializationDurationMs) {
// Do nothing.
}
/**
* Called when a new {@link Format} is read from the upstream {@link MediaPeriod}.
*
* @param formatHolder A {@link FormatHolder} that holds the new {@link Format}.
* @throws ExoPlaybackException If an error occurs re-initializing the {@link MediaCodec}.
*/
@SuppressWarnings("unchecked")
protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybackException {
waitingForFirstSampleInFormat = true;
Format newFormat = Assertions.checkNotNull(formatHolder.format);
if (formatHolder.includesDrmSession) {
setSourceDrmSession((DrmSession The default implementation is a no-op.
*
* @param codec The {@link MediaCodec} instance.
* @param outputMediaFormat The new output {@link MediaFormat}.
* @throws ExoPlaybackException Thrown if an error occurs handling the new output media format.
*/
protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
throws ExoPlaybackException {
// Do nothing.
}
/**
* Handles supplemental data associated with an input buffer.
*
* The default implementation is a no-op.
*
* @param buffer The input buffer that is about to be queued.
* @throws ExoPlaybackException Thrown if an error occurs handling supplemental data.
*/
protected void handleInputBufferSupplementalData(DecoderInputBuffer buffer)
throws ExoPlaybackException {
// Do nothing.
}
/**
* Called immediately before an input buffer is queued into the codec.
*
* The default implementation is a no-op.
*
* @param buffer The buffer to be queued.
*/
protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
// Do nothing.
}
/**
* Called when an output buffer is successfully processed.
*
* The default implementation is a no-op.
*
* @param presentationTimeUs The timestamp associated with the output buffer.
*/
protected void onProcessedOutputBuffer(long presentationTimeUs) {
// Do nothing.
}
/**
* Determines whether the existing {@link MediaCodec} can be kept for a new {@link Format}, and if
* it can whether it requires reconfiguration.
*
* The default implementation returns {@link #KEEP_CODEC_RESULT_NO}.
*
* @param codec The existing {@link MediaCodec} instance.
* @param codecInfo A {@link MediaCodecInfo} describing the decoder.
* @param oldFormat The {@link Format} for which the existing instance is configured.
* @param newFormat The new {@link Format}.
* @return Whether the instance can be kept, and if it can whether it requires reconfiguration.
*/
protected @KeepCodecResult int canKeepCodec(
MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) {
return KEEP_CODEC_RESULT_NO;
}
@Override
public boolean isEnded() {
return outputStreamEnded;
}
@Override
public boolean isReady() {
return inputFormat != null
&& !waitingForKeys
&& (isSourceReady()
|| hasOutputBuffer()
|| (codecHotswapDeadlineMs != C.TIME_UNSET
&& SystemClock.elapsedRealtime() < codecHotswapDeadlineMs));
}
/**
* Returns the maximum time to block whilst waiting for a decoded output buffer.
*
* @return The maximum time to block, in microseconds.
*/
protected long getDequeueOutputBufferTimeoutUs() {
return 0;
}
/**
* Returns the {@link MediaFormat#KEY_OPERATING_RATE} value for a given renderer operating rate,
* current {@link Format} and set of possible stream formats.
*
* The default implementation returns {@link #CODEC_OPERATING_RATE_UNSET}.
*
* @param operatingRate The renderer operating rate.
* @param format The {@link Format} for which the codec is being configured.
* @param streamFormats The possible stream formats.
* @return The codec operating rate, or {@link #CODEC_OPERATING_RATE_UNSET} if no codec operating
* rate should be set.
*/
protected float getCodecOperatingRateV23(
float operatingRate, Format format, Format[] streamFormats) {
return CODEC_OPERATING_RATE_UNSET;
}
/**
* Updates the codec operating rate.
*
* @throws ExoPlaybackException If an error occurs releasing or initializing a codec.
*/
private void updateCodecOperatingRate() throws ExoPlaybackException {
if (Util.SDK_INT < 23) {
return;
}
float newCodecOperatingRate =
getCodecOperatingRateV23(rendererOperatingRate, codecFormat, getStreamFormats());
if (codecOperatingRate == newCodecOperatingRate) {
// No change.
} else if (newCodecOperatingRate == CODEC_OPERATING_RATE_UNSET) {
// The only way to clear the operating rate is to instantiate a new codec instance. See
// [Internal ref: b/71987865].
drainAndReinitializeCodec();
} else if (codecOperatingRate != CODEC_OPERATING_RATE_UNSET
|| newCodecOperatingRate > assumedMinimumCodecOperatingRate) {
// We need to set the operating rate, either because we've set it previously or because it's
// above the assumed minimum rate.
Bundle codecParameters = new Bundle();
codecParameters.putFloat(MediaFormat.KEY_OPERATING_RATE, newCodecOperatingRate);
codec.setParameters(codecParameters);
codecOperatingRate = newCodecOperatingRate;
}
}
/** Starts draining the codec for flush. */
private void drainAndFlushCodec() {
if (codecReceivedBuffers) {
codecDrainState = DRAIN_STATE_SIGNAL_END_OF_STREAM;
codecDrainAction = DRAIN_ACTION_FLUSH;
}
}
/**
* Starts draining the codec to update its DRM session. The update may occur immediately if no
* buffers have been queued to the codec.
*
* @throws ExoPlaybackException If an error occurs updating the codec's DRM session.
*/
private void drainAndUpdateCodecDrmSession() throws ExoPlaybackException {
if (Util.SDK_INT < 23) {
// The codec needs to be re-initialized to switch to the source DRM session.
drainAndReinitializeCodec();
return;
}
if (codecReceivedBuffers) {
codecDrainState = DRAIN_STATE_SIGNAL_END_OF_STREAM;
codecDrainAction = DRAIN_ACTION_UPDATE_DRM_SESSION;
} else {
// Nothing has been queued to the decoder, so we can do the update immediately.
updateDrmSessionOrReinitializeCodecV23();
}
}
/**
* Starts draining the codec for re-initialization. Re-initialization may occur immediately if no
* buffers have been queued to the codec.
*
* @throws ExoPlaybackException If an error occurs re-initializing a codec.
*/
private void drainAndReinitializeCodec() throws ExoPlaybackException {
if (codecReceivedBuffers) {
codecDrainState = DRAIN_STATE_SIGNAL_END_OF_STREAM;
codecDrainAction = DRAIN_ACTION_REINITIALIZE;
} else {
// Nothing has been queued to the decoder, so we can re-initialize immediately.
reinitializeCodec();
}
}
/**
* @return Whether it may be possible to drain more output data.
* @throws ExoPlaybackException If an error occurs draining the output buffer.
*/
private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
throws ExoPlaybackException {
if (!hasOutputBuffer()) {
int outputIndex;
if (codecNeedsEosOutputExceptionWorkaround && codecReceivedEos) {
try {
outputIndex =
codec.dequeueOutputBuffer(outputBufferInfo, getDequeueOutputBufferTimeoutUs());
} catch (IllegalStateException e) {
processEndOfStream();
if (outputStreamEnded) {
// Release the codec, as it's in an error state.
releaseCodec();
}
return false;
}
} else {
outputIndex =
codec.dequeueOutputBuffer(outputBufferInfo, getDequeueOutputBufferTimeoutUs());
}
if (outputIndex < 0) {
if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
processOutputFormat();
return true;
} else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
processOutputBuffersChanged();
return true;
}
/* MediaCodec.INFO_TRY_AGAIN_LATER (-1) or unknown negative return value */
if (codecNeedsEosPropagation
&& (inputStreamEnded || codecDrainState == DRAIN_STATE_WAIT_END_OF_STREAM)) {
processEndOfStream();
}
return false;
}
// We've dequeued a buffer.
if (shouldSkipAdaptationWorkaroundOutputBuffer) {
shouldSkipAdaptationWorkaroundOutputBuffer = false;
codec.releaseOutputBuffer(outputIndex, false);
return true;
} else if (outputBufferInfo.size == 0
&& (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
// The dequeued buffer indicates the end of the stream. Process it immediately.
processEndOfStream();
return false;
}
this.outputIndex = outputIndex;
outputBuffer = getOutputBuffer(outputIndex);
// The dequeued buffer is a media buffer. Do some initial setup.
// It will be processed by calling processOutputBuffer (possibly multiple times).
if (outputBuffer != null) {
outputBuffer.position(outputBufferInfo.offset);
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
}
isDecodeOnlyOutputBuffer = isDecodeOnlyBuffer(outputBufferInfo.presentationTimeUs);
isLastOutputBuffer =
lastBufferInStreamPresentationTimeUs == outputBufferInfo.presentationTimeUs;
updateOutputFormatForTime(outputBufferInfo.presentationTimeUs);
}
boolean processedOutputBuffer;
if (codecNeedsEosOutputExceptionWorkaround && codecReceivedEos) {
try {
processedOutputBuffer =
processOutputBuffer(
positionUs,
elapsedRealtimeUs,
codec,
outputBuffer,
outputIndex,
outputBufferInfo.flags,
outputBufferInfo.presentationTimeUs,
isDecodeOnlyOutputBuffer,
isLastOutputBuffer,
outputFormat);
} catch (IllegalStateException e) {
processEndOfStream();
if (outputStreamEnded) {
// Release the codec, as it's in an error state.
releaseCodec();
}
return false;
}
} else {
processedOutputBuffer =
processOutputBuffer(
positionUs,
elapsedRealtimeUs,
codec,
outputBuffer,
outputIndex,
outputBufferInfo.flags,
outputBufferInfo.presentationTimeUs,
isDecodeOnlyOutputBuffer,
isLastOutputBuffer,
outputFormat);
}
if (processedOutputBuffer) {
onProcessedOutputBuffer(outputBufferInfo.presentationTimeUs);
boolean isEndOfStream = (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
resetOutputBuffer();
if (!isEndOfStream) {
return true;
}
processEndOfStream();
}
return false;
}
/** Processes a new output {@link MediaFormat}. */
private void processOutputFormat() throws ExoPlaybackException {
codecHasOutputMediaFormat = true;
MediaFormat mediaFormat = codec.getOutputFormat();
if (codecAdaptationWorkaroundMode != ADAPTATION_WORKAROUND_MODE_NEVER
&& mediaFormat.getInteger(MediaFormat.KEY_WIDTH) == ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT
&& mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)
== ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT) {
// We assume this format changed event was caused by the adaptation workaround.
shouldSkipAdaptationWorkaroundOutputBuffer = true;
return;
}
if (codecNeedsMonoChannelCountWorkaround) {
mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
}
onOutputFormatChanged(codec, mediaFormat);
}
/**
* Processes a change in the output buffers.
*/
private void processOutputBuffersChanged() {
if (Util.SDK_INT < 21) {
outputBuffers = codec.getOutputBuffers();
}
}
/**
* Processes an output media buffer.
*
* When a new {@link ByteBuffer} is passed to this method its position and limit delineate the
* data to be processed. The return value indicates whether the buffer was processed in full. If
* true is returned then the next call to this method will receive a new buffer to be processed.
* If false is returned then the same buffer will be passed to the next call. An implementation of
* this method is free to modify the buffer and can assume that the buffer will not be externally
* modified between successive calls. Hence an implementation can, for example, modify the
* buffer's position to keep track of how much of the data it has processed.
*
* Note that the first call to this method following a call to {@link #onPositionReset(long,
* boolean)} will always receive a new {@link ByteBuffer} to be processed.
*
* @param positionUs The current media time in microseconds, measured at the start of the current
* iteration of the rendering loop.
* @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the
* start of the current iteration of the rendering loop.
* @param codec The {@link MediaCodec} instance.
* @param buffer The output buffer to process.
* @param bufferIndex The index of the output buffer.
* @param bufferFlags The flags attached to the output buffer.
* @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds.
* @param isDecodeOnlyBuffer Whether the buffer was marked with {@link C#BUFFER_FLAG_DECODE_ONLY}
* by the source.
* @param isLastBuffer Whether the buffer is the last sample of the current stream.
* @param format The {@link Format} associated with the buffer.
* @return Whether the output buffer was fully processed (e.g. rendered or skipped).
* @throws ExoPlaybackException If an error occurs processing the output buffer.
*/
protected abstract boolean processOutputBuffer(
long positionUs,
long elapsedRealtimeUs,
MediaCodec codec,
ByteBuffer buffer,
int bufferIndex,
int bufferFlags,
long bufferPresentationTimeUs,
boolean isDecodeOnlyBuffer,
boolean isLastBuffer,
Format format)
throws ExoPlaybackException;
/**
* Incrementally renders any remaining output.
*
* The default implementation is a no-op.
*
* @throws ExoPlaybackException Thrown if an error occurs rendering remaining output.
*/
protected void renderToEndOfStream() throws ExoPlaybackException {
// Do nothing.
}
/**
* Processes an end of stream signal.
*
* @throws ExoPlaybackException If an error occurs processing the signal.
*/
private void processEndOfStream() throws ExoPlaybackException {
switch (codecDrainAction) {
case DRAIN_ACTION_REINITIALIZE:
reinitializeCodec();
break;
case DRAIN_ACTION_UPDATE_DRM_SESSION:
updateDrmSessionOrReinitializeCodecV23();
break;
case DRAIN_ACTION_FLUSH:
flushOrReinitializeCodec();
break;
case DRAIN_ACTION_NONE:
default:
outputStreamEnded = true;
renderToEndOfStream();
break;
}
}
/**
* Notifies the renderer that output end of stream is pending and should be handled on the next
* render.
*/
protected final void setPendingOutputEndOfStream() {
pendingOutputEndOfStream = true;
}
private void reinitializeCodec() throws ExoPlaybackException {
releaseCodec();
maybeInitCodec();
}
private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
// We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
// box presentationTimeUs, creating a Long object that would need to be garbage collected.
int size = decodeOnlyPresentationTimestamps.size();
for (int i = 0; i < size; i++) {
if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
decodeOnlyPresentationTimestamps.remove(i);
return true;
}
}
return false;
}
@TargetApi(23)
private void updateDrmSessionOrReinitializeCodecV23() throws ExoPlaybackException {
@Nullable FrameworkMediaCrypto sessionMediaCrypto = sourceDrmSession.getMediaCrypto();
if (sessionMediaCrypto == null) {
// We'd only expect this to happen if the CDM from which the pending session is obtained needs
// provisioning. This is unlikely to happen (it probably requires a switch from one DRM scheme
// to another, where the new CDM hasn't been used before and needs provisioning). It would be
// possible to handle this case more efficiently (i.e. with a new renderer state that waits
// for provisioning to finish and then calls mediaCrypto.setMediaDrmSession), but the extra
// complexity is not warranted given how unlikely the case is to occur.
reinitializeCodec();
return;
}
if (C.PLAYREADY_UUID.equals(sessionMediaCrypto.uuid)) {
// The PlayReady CDM does not implement setMediaDrmSession.
// TODO: Add API check once [Internal ref: b/128835874] is fixed.
reinitializeCodec();
return;
}
if (flushOrReinitializeCodec()) {
// The codec was reinitialized. The new codec will be using the new DRM session, so there's
// nothing more to do.
return;
}
try {
mediaCrypto.setMediaDrmSession(sessionMediaCrypto.sessionId);
} catch (MediaCryptoException e) {
throw createRendererException(e, inputFormat);
}
setCodecDrmSession(sourceDrmSession);
codecDrainState = DRAIN_STATE_NONE;
codecDrainAction = DRAIN_ACTION_NONE;
}
/**
* Returns whether a {@link DrmSession} may require a secure decoder for a given {@link Format}.
*
* @param drmSession The {@link DrmSession}.
* @param format The {@link Format}.
* @return Whether a secure decoder may be required.
*/
private static boolean maybeRequiresSecureDecoder(
DrmSession
* If true is returned, the renderer will work around the issue by releasing the decoder and
* instantiating a new one rather than flushing the current instance.
*
* See [Internal: b/8347958, b/8543366].
*
* @param name The name of the decoder.
* @return True if the decoder is known to fail when flushed.
*/
private static boolean codecNeedsFlushWorkaround(String name) {
return Util.SDK_INT < 18
|| (Util.SDK_INT == 18
&& ("OMX.SEC.avc.dec".equals(name) || "OMX.SEC.avc.dec.secure".equals(name)))
|| (Util.SDK_INT == 19 && Util.MODEL.startsWith("SM-G800")
&& ("OMX.Exynos.avc.dec".equals(name) || "OMX.Exynos.avc.dec.secure".equals(name)));
}
/**
* Returns a mode that specifies when the adaptation workaround should be enabled.
*
* When enabled, the workaround queues and discards a blank frame with a resolution whose width
* and height both equal {@link #ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT}, to reset the decoder's
* internal state when a format change occurs.
*
* See [Internal: b/27807182]. See GitHub issue #3257.
*
* @param name The name of the decoder.
* @return The mode specifying when the adaptation workaround should be enabled.
*/
private @AdaptationWorkaroundMode int codecAdaptationWorkaroundMode(String name) {
if (Util.SDK_INT <= 25 && "OMX.Exynos.avc.dec.secure".equals(name)
&& (Util.MODEL.startsWith("SM-T585") || Util.MODEL.startsWith("SM-A510")
|| Util.MODEL.startsWith("SM-A520") || Util.MODEL.startsWith("SM-J700"))) {
return ADAPTATION_WORKAROUND_MODE_ALWAYS;
} else if (Util.SDK_INT < 24
&& ("OMX.Nvidia.h264.decode".equals(name) || "OMX.Nvidia.h264.decode.secure".equals(name))
&& ("flounder".equals(Util.DEVICE) || "flounder_lte".equals(Util.DEVICE)
|| "grouper".equals(Util.DEVICE) || "tilapia".equals(Util.DEVICE))) {
return ADAPTATION_WORKAROUND_MODE_SAME_RESOLUTION;
} else {
return ADAPTATION_WORKAROUND_MODE_NEVER;
}
}
/**
* Returns whether the decoder is known to fail when an attempt is made to reconfigure it with a
* new format's configuration data.
*
* When enabled, the workaround will always release and recreate the decoder, rather than
* attempting to reconfigure the existing instance.
*
* @param name The name of the decoder.
* @return True if the decoder is known to fail when an attempt is made to reconfigure it with a
* new format's configuration data.
*/
private static boolean codecNeedsReconfigureWorkaround(String name) {
return Util.MODEL.startsWith("SM-T230") && "OMX.MARVELL.VIDEO.HW.CODA7542DECODER".equals(name);
}
/**
* Returns whether the decoder is an H.264/AVC decoder known to fail if NAL units are queued
* before the codec specific data.
*
* If true is returned, the renderer will work around the issue by discarding data up to the
* SPS.
*
* @param name The name of the decoder.
* @param format The {@link Format} used to configure the decoder.
* @return True if the decoder is known to fail if NAL units are queued before CSD.
*/
private static boolean codecNeedsDiscardToSpsWorkaround(String name, Format format) {
return Util.SDK_INT < 21 && format.initializationData.isEmpty()
&& "OMX.MTK.VIDEO.DECODER.AVC".equals(name);
}
/**
* Returns whether the decoder is known to handle the propagation of the {@link
* MediaCodec#BUFFER_FLAG_END_OF_STREAM} flag incorrectly on the host device.
*
* If true is returned, the renderer will work around the issue by approximating end of stream
* behavior without relying on the flag being propagated through to an output buffer by the
* underlying decoder.
*
* @param codecInfo Information about the {@link MediaCodec}.
* @return True if the decoder is known to handle {@link MediaCodec#BUFFER_FLAG_END_OF_STREAM}
* propagation incorrectly on the host device. False otherwise.
*/
private static boolean codecNeedsEosPropagationWorkaround(MediaCodecInfo codecInfo) {
String name = codecInfo.name;
return (Util.SDK_INT <= 25 && "OMX.rk.video_decoder.avc".equals(name))
|| (Util.SDK_INT <= 17 && "OMX.allwinner.video.decoder.avc".equals(name))
|| ("Amazon".equals(Util.MANUFACTURER) && "AFTS".equals(Util.MODEL) && codecInfo.secure);
}
/**
* Returns whether the decoder is known to behave incorrectly if flushed after receiving an input
* buffer with {@link MediaCodec#BUFFER_FLAG_END_OF_STREAM} set.
*
* If true is returned, the renderer will work around the issue by instantiating a new decoder
* when this case occurs.
*
* See [Internal: b/8578467, b/23361053].
*
* @param name The name of the decoder.
* @return True if the decoder is known to behave incorrectly if flushed after receiving an input
* buffer with {@link MediaCodec#BUFFER_FLAG_END_OF_STREAM} set. False otherwise.
*/
private static boolean codecNeedsEosFlushWorkaround(String name) {
return (Util.SDK_INT <= 23 && "OMX.google.vorbis.decoder".equals(name))
|| (Util.SDK_INT <= 19
&& ("hb2000".equals(Util.DEVICE) || "stvm8".equals(Util.DEVICE))
&& ("OMX.amlogic.avc.decoder.awesome".equals(name)
|| "OMX.amlogic.avc.decoder.awesome.secure".equals(name)));
}
/**
* Returns whether the decoder may throw an {@link IllegalStateException} from
* {@link MediaCodec#dequeueOutputBuffer(MediaCodec.BufferInfo, long)} or
* {@link MediaCodec#releaseOutputBuffer(int, boolean)} after receiving an input
* buffer with {@link MediaCodec#BUFFER_FLAG_END_OF_STREAM} set.
*
* See [Internal: b/17933838].
*
* @param name The name of the decoder.
* @return True if the decoder may throw an exception after receiving an end-of-stream buffer.
*/
private static boolean codecNeedsEosOutputExceptionWorkaround(String name) {
return Util.SDK_INT == 21 && "OMX.google.aac.decoder".equals(name);
}
/**
* Returns whether the decoder is known to set the number of audio channels in the output {@link
* Format} to 2 for the given input {@link Format}, whilst only actually outputting a single
* channel.
*
* If true is returned then we explicitly override the number of channels in the output {@link
* Format}, setting it to 1.
*
* @param name The decoder name.
* @param format The input {@link Format}.
* @return True if the decoder is known to set the number of audio channels in the output {@link
* Format} to 2 for the given input {@link Format}, whilst only actually outputting a single
* channel. False otherwise.
*/
private static boolean codecNeedsMonoChannelCountWorkaround(String name, Format format) {
return Util.SDK_INT <= 18 && format.channelCount == 1
&& "OMX.MTK.AUDIO.DECODER.MP3".equals(name);
}
/**
* Returns whether the decoder is known to behave incorrectly if flushed prior to having output a
* {@link MediaFormat}.
*
* If true is returned, the renderer will work around the issue by instantiating a new decoder
* when this case occurs.
*
* See [Internal: b/141097367].
*
* @param name The name of the decoder.
* @return True if the decoder is known to behave incorrectly if flushed prior to having output a
* {@link MediaFormat}. False otherwise.
*/
private static boolean codecNeedsSosFlushWorkaround(String name) {
return Util.SDK_INT == 29 && "c2.android.aac.decoder".equals(name);
}
}