/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package org.mozilla.gecko.media; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; import android.media.MediaCodec.CryptoInfo; import android.util.Log; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import org.mozilla.geckoview.BuildConfig; import org.mozilla.thirdparty.com.google.android.exoplayer2.C; import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecInfo; import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecSelector; import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecUtil; import org.mozilla.thirdparty.com.google.android.exoplayer2.util.MimeTypes; public class GeckoHlsVideoRenderer extends GeckoHlsRendererBase { /* * By configuring these states, initialization data is provided for * ExoPlayer's HlsMediaSource to parse HLS bitstream and then provide samples * starting with an Access Unit Delimiter including SPS/PPS for TS, * and provide samples starting with an AUD without SPS/PPS for FMP4. */ private enum RECONFIGURATION_STATE { NONE, WRITE_PENDING, QUEUE_PENDING } private boolean mRendererReconfigured; private RECONFIGURATION_STATE mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; // A list of the formats which may be included in the bitstream. private Format[] mStreamFormats; // The max width/height/inputBufferSize for specific codec format. private CodecMaxValues mCodecMaxValues; // A temporary queue for samples whose duration is not calculated yet. private ConcurrentLinkedQueue mDemuxedNoDurationSamples = new ConcurrentLinkedQueue<>(); // Contain CSD-0(SPS)/CSD-1(PPS) information (in AnnexB format) for // prepending each keyframe. When video format changes, this information // changes accordingly. private byte[] mCSDInfo = null; public GeckoHlsVideoRenderer(final GeckoHlsPlayer.ComponentEventDispatcher eventDispatcher) { super(C.TRACK_TYPE_VIDEO, eventDispatcher); LOGTAG = getClass().getSimpleName(); DEBUG = !BuildConfig.MOZILLA_OFFICIAL; } @Override public final int supportsMixedMimeTypeAdaptation() { return ADAPTIVE_NOT_SEAMLESS; } @Override public final int supportsFormat(final Format format) { /* * FORMAT_EXCEEDS_CAPABILITIES : The Renderer is capable of rendering * formats with the same mime type, but * the properties of the format exceed * the renderer's capability. * FORMAT_UNSUPPORTED_SUBTYPE : The Renderer is a general purpose * renderer for formats of the same * top-level type, but is not capable of * rendering the format or any other format * with the same mime type because the * sub-type is not supported. * FORMAT_UNSUPPORTED_TYPE : The Renderer is not capable of rendering * the format, either because it does not support * the format's top-level type, or because it's * a specialized renderer for a different mime type. * ADAPTIVE_NOT_SEAMLESS : The Renderer can adapt between formats, * but may suffer a brief discontinuity (~50-100ms) * when adaptation occurs. * ADAPTIVE_SEAMLESS : The Renderer can seamlessly adapt between formats. */ final String mimeType = format.sampleMimeType; if (!MimeTypes.isVideo(mimeType)) { return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); } List decoderInfos = null; try { final MediaCodecSelector mediaCodecSelector = MediaCodecSelector.DEFAULT; decoderInfos = mediaCodecSelector.getDecoderInfos(mimeType, false, false); } catch (final MediaCodecUtil.DecoderQueryException e) { Log.e(LOGTAG, e.getMessage()); } if (decoderInfos == null || decoderInfos.isEmpty()) { return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE); } boolean decoderCapable = false; MediaCodecInfo info = null; for (final MediaCodecInfo i : decoderInfos) { if (i.isCodecSupported(format)) { decoderCapable = true; info = i; } } if (decoderCapable && format.width > 0 && format.height > 0) { decoderCapable = info.isVideoSizeAndRateSupportedV21(format.width, format.height, format.frameRate); } return RendererCapabilities.create( decoderCapable ? FORMAT_HANDLED : FORMAT_EXCEEDS_CAPABILITIES, info != null && info.adaptive ? ADAPTIVE_SEAMLESS : ADAPTIVE_NOT_SEAMLESS, TUNNELING_NOT_SUPPORTED); } @Override protected final void createInputBuffer() throws ExoPlaybackException { assertTrue(mFormats.size() > 0); // Calculate maximum size which might be used for target format. final Format currentFormat = mFormats.get(mFormats.size() - 1); mCodecMaxValues = getCodecMaxValues(currentFormat, mStreamFormats); // Create a buffer with maximal size for reading source. // Note : Though we are able to dynamically enlarge buffer size by // creating DecoderInputBuffer with specific BufferReplacementMode, we // still allocate a calculated max size buffer for it at first to reduce // runtime overhead. try { mInputBuffer = ByteBuffer.wrap(new byte[mCodecMaxValues.inputSize]); } catch (final OutOfMemoryError e) { Log.e(LOGTAG, "cannot allocate input buffer of size " + mCodecMaxValues.inputSize, e); throw ExoPlaybackException.createForRenderer( new Exception(e), getIndex(), mFormats.isEmpty() ? null : getFormat(mFormats.size() - 1), RendererCapabilities.FORMAT_HANDLED); } } @Override protected void resetRenderer() { if (DEBUG) { Log.d(LOGTAG, "[resetRenderer] mInitialized = " + mInitialized); } if (mInitialized) { mRendererReconfigured = false; mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; mInputBuffer = null; mCSDInfo = null; mInitialized = false; } } @Override protected void handleReconfiguration(final DecoderInputBuffer bufferForRead) { // For adaptive reconfiguration OMX decoders expect all reconfiguration // data to be supplied at the start of the buffer that also contains // the first frame in the new format. assertTrue(mFormats.size() > 0); if (mRendererReconfigurationState == RECONFIGURATION_STATE.WRITE_PENDING) { if (bufferForRead.data == null) { if (DEBUG) { Log.d(LOGTAG, "[feedInput][WRITE_PENDING] bufferForRead.data is not initialized."); } return; } if (DEBUG) { Log.d(LOGTAG, "[feedInput][WRITE_PENDING] put initialization data"); } final Format currentFormat = mFormats.get(mFormats.size() - 1); for (int i = 0; i < currentFormat.initializationData.size(); i++) { final byte[] data = currentFormat.initializationData.get(i); bufferForRead.data.put(data); } mRendererReconfigurationState = RECONFIGURATION_STATE.QUEUE_PENDING; } } @Override protected void handleFormatRead(final DecoderInputBuffer bufferForRead) throws ExoPlaybackException { if (mRendererReconfigurationState == RECONFIGURATION_STATE.QUEUE_PENDING) { if (DEBUG) { Log.d(LOGTAG, "[feedInput][QUEUE_PENDING] 2 formats in a row."); } // We received two formats in a row. Clear the current buffer of any reconfiguration data // associated with the first format. bufferForRead.clear(); mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; } onInputFormatChanged(mFormatHolder.format); } @Override protected void handleEndOfStream(final DecoderInputBuffer bufferForRead) { if (mRendererReconfigurationState == RECONFIGURATION_STATE.QUEUE_PENDING) { if (DEBUG) { Log.d(LOGTAG, "[feedInput][QUEUE_PENDING] isEndOfStream."); } // We received a new format immediately before the end of the stream. We need to clear // the corresponding reconfiguration data from the current buffer, but re-write it into // a subsequent buffer if there are any (e.g. if the user seeks backwards). bufferForRead.clear(); mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; } mInputStreamEnded = true; final GeckoHLSSample sample = GeckoHLSSample.EOS; calculatDuration(sample); } @Override protected void handleSamplePreparation(final DecoderInputBuffer bufferForRead) { final int csdInfoSize = mCSDInfo != null ? mCSDInfo.length : 0; final int dataSize = bufferForRead.data.limit(); final int size = bufferForRead.isKeyFrame() ? csdInfoSize + dataSize : dataSize; final byte[] realData = new byte[size]; if (bufferForRead.isKeyFrame()) { // Prepend the CSD information to the sample if it's a key frame. System.arraycopy(mCSDInfo, 0, realData, 0, csdInfoSize); bufferForRead.data.get(realData, csdInfoSize, dataSize); } else { bufferForRead.data.get(realData, 0, dataSize); } final ByteBuffer buffer = ByteBuffer.wrap(realData); mInputBuffer = bufferForRead.data; mInputBuffer.clear(); final CryptoInfo cryptoInfo = bufferForRead.isEncrypted() ? bufferForRead.cryptoInfo.getFrameworkCryptoInfoV16() : null; final BufferInfo bufferInfo = new BufferInfo(); // Flags in DecoderInputBuffer are synced with MediaCodec Buffer flags. int flags = 0; flags |= bufferForRead.isKeyFrame() ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0; flags |= bufferForRead.isEndOfStream() ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0; bufferInfo.set(0, size, bufferForRead.timeUs, flags); assertTrue(mFormats.size() > 0); // We add a new format in the list once format changes, so the formatIndex // should indicate to the last(latest) format. final GeckoHLSSample sample = GeckoHLSSample.create(buffer, bufferInfo, cryptoInfo, mFormats.size() - 1); // There's no duration information from the ExoPlayer's sample, we need // to calculate it. calculatDuration(sample); mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; } @Override protected void onPositionReset(final long positionUs, final boolean joining) { super.onPositionReset(positionUs, joining); if (mInitialized && mRendererReconfigured && mFormats.size() != 0) { if (DEBUG) { Log.d(LOGTAG, "[onPositionReset] WRITE_PENDING"); } // Any reconfiguration data that we put shortly before the reset // may be invalid. We avoid this issue by sending reconfiguration // data following every position reset. mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; } } @Override protected boolean clearInputSamplesQueue() { if (DEBUG) { Log.d(LOGTAG, "clearInputSamplesQueue"); } mDemuxedInputSamples.clear(); mDemuxedNoDurationSamples.clear(); return true; } @Override protected boolean canReconfigure(final Format oldFormat, final Format newFormat) { final boolean canReconfig = areAdaptationCompatible(oldFormat, newFormat) && newFormat.width <= mCodecMaxValues.width && newFormat.height <= mCodecMaxValues.height && newFormat.maxInputSize <= mCodecMaxValues.inputSize; if (DEBUG) { Log.d(LOGTAG, "[canReconfigure] : " + canReconfig); } return canReconfig; } @Override protected void prepareReconfiguration() { if (DEBUG) { Log.d(LOGTAG, "[onInputFormatChanged] starting reconfiguration !"); } mRendererReconfigured = true; mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; } @Override protected void updateCSDInfo(final Format format) { int size = 0; for (int i = 0; i < format.initializationData.size(); i++) { size += format.initializationData.get(i).length; } int startPos = 0; mCSDInfo = new byte[size]; for (int i = 0; i < format.initializationData.size(); i++) { final byte[] data = format.initializationData.get(i); System.arraycopy(data, 0, mCSDInfo, startPos, data.length); startPos += data.length; } if (DEBUG) { Log.d(LOGTAG, "mCSDInfo [" + Utils.bytesToHex(mCSDInfo) + "]"); } } @Override protected void notifyPlayerInputFormatChanged(final Format newFormat) { mPlayerEventDispatcher.onVideoInputFormatChanged(newFormat); } private void calculateSamplesWithin(final GeckoHLSSample[] samples, final int range) { // Calculate the first 'range' elements. for (int i = 0; i < range; i++) { // Comparing among samples in the window. for (int j = -2; j < 14; j++) { if (i + j >= 0 && i + j < range && samples[i + j].info.presentationTimeUs > samples[i].info.presentationTimeUs) { samples[i].duration = Math.min( samples[i].duration, samples[i + j].info.presentationTimeUs - samples[i].info.presentationTimeUs); } } } } private void calculatDuration(final GeckoHLSSample inputSample) { /* * NOTE : * Since we customized renderer as a demuxer. Here we're not able to * obtain duration from the DecoderInputBuffer as there's no duration inside. * So we calcualte it by referring to nearby samples' timestamp. * A temporary queue |mDemuxedNoDurationSamples| is used to queue demuxed * samples from HlsMediaSource which have no duration information at first. * We're choosing 16 as the comparing window size, because it's commonly * used as a GOP size. * Considering there're 16 demuxed samples in the _no duration_ queue already, * e.g. |-2|-1|0|1|2|3|4|5|6|...|13| * Once a new demuxed(No duration) sample X (17th) is put into the * temporary queue, * e.g. |-2|-1|0|1|2|3|4|5|6|...|13|X| * we are able to calculate the correct duration for sample 0 by finding * the closest but greater pts than sample 0 among these 16 samples, * here, let's say sample -2 to 13. */ if (inputSample != null) { mDemuxedNoDurationSamples.offer(inputSample); } final int sizeOfNoDura = mDemuxedNoDurationSamples.size(); // A calculation window we've ever found suitable for both HLS TS & FMP4. final int range = sizeOfNoDura >= 17 ? 17 : sizeOfNoDura; final GeckoHLSSample[] inputArray = mDemuxedNoDurationSamples.toArray(new GeckoHLSSample[sizeOfNoDura]); if (range >= 17 && !mInputStreamEnded) { calculateSamplesWithin(inputArray, range); final GeckoHLSSample toQueue = mDemuxedNoDurationSamples.poll(); mDemuxedInputSamples.offer(toQueue); if (BuildConfig.DEBUG_BUILD) { Log.d( LOGTAG, "Demuxed sample PTS : " + toQueue.info.presentationTimeUs + ", duration :" + toQueue.duration + ", isKeyFrame(" + toQueue.isKeyFrame() + ", formatIndex(" + toQueue.formatIndex + "), queue size : " + mDemuxedInputSamples.size() + ", NoDuQueue size : " + mDemuxedNoDurationSamples.size()); } } else if (mInputStreamEnded) { calculateSamplesWithin(inputArray, sizeOfNoDura); // NOTE : We're not able to calculate the duration for the last sample. // A workaround here is to assign a close duration to it. long prevDuration = 33333; GeckoHLSSample sample = null; for (sample = mDemuxedNoDurationSamples.poll(); sample != null; sample = mDemuxedNoDurationSamples.poll()) { if (sample.duration == Long.MAX_VALUE) { sample.duration = prevDuration; if (DEBUG) { Log.d(LOGTAG, "Adjust the PTS of the last sample to " + sample.duration + " (us)"); } } prevDuration = sample.duration; if (DEBUG) { Log.d( LOGTAG, "last loop to offer samples - PTS : " + sample.info.presentationTimeUs + ", Duration : " + sample.duration + ", isEOS : " + sample.isEOS()); } mDemuxedInputSamples.offer(sample); } } } // Return the time of first keyframe sample in the queue. // If there's no key frame in the queue, return the MAX_VALUE so // MFR won't mistake for that which the decode is getting slow. public long getNextKeyFrameTime() { long nextKeyFrameTime = Long.MAX_VALUE; for (final GeckoHLSSample sample : mDemuxedInputSamples) { if (sample != null && (sample.info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { nextKeyFrameTime = sample.info.presentationTimeUs; break; } } return nextKeyFrameTime; } @Override protected void onStreamChanged(final Format[] formats, final long offsetUs) { mStreamFormats = formats; } private static CodecMaxValues getCodecMaxValues( final Format format, final Format[] streamFormats) { int maxWidth = format.width; int maxHeight = format.height; int maxInputSize = getMaxInputSize(format); for (final Format streamFormat : streamFormats) { if (areAdaptationCompatible(format, streamFormat)) { maxWidth = Math.max(maxWidth, streamFormat.width); maxHeight = Math.max(maxHeight, streamFormat.height); maxInputSize = Math.max(maxInputSize, getMaxInputSize(streamFormat)); } } return new CodecMaxValues(maxWidth, maxHeight, maxInputSize); } private static int getMaxInputSize(final Format format) { if (format.maxInputSize != Format.NO_VALUE) { // The format defines an explicit maximum input size. return format.maxInputSize; } if (format.width == Format.NO_VALUE || format.height == Format.NO_VALUE) { // We can't infer a maximum input size without video dimensions. return Format.NO_VALUE; } // Attempt to infer a maximum input size from the format. final int maxPixels; final int minCompressionRatio; switch (format.sampleMimeType) { case MimeTypes.VIDEO_H264: // Round up width/height to an integer number of macroblocks. maxPixels = ((format.width + 15) / 16) * ((format.height + 15) / 16) * 16 * 16; minCompressionRatio = 2; break; default: // Leave the default max input size. return Format.NO_VALUE; } // Estimate the maximum input size assuming three channel 4:2:0 subsampled input frames. return (maxPixels * 3) / (2 * minCompressionRatio); } private static boolean areAdaptationCompatible(final Format first, final Format second) { return first.sampleMimeType.equals(second.sampleMimeType) && getRotationDegrees(first) == getRotationDegrees(second); } private static int getRotationDegrees(final Format format) { return format.rotationDegrees == Format.NO_VALUE ? 0 : format.rotationDegrees; } private static final class CodecMaxValues { public final int width; public final int height; public final int inputSize; public CodecMaxValues(final int width, final int height, final int inputSize) { this.width = width; this.height = height; this.inputSize = inputSize; } } }