From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- .../main/java/org/mozilla/gecko/media/Codec.java | 713 +++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java (limited to 'mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java') diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java new file mode 100644 index 0000000000..eb07f6146c --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java @@ -0,0 +1,713 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.VideoCapabilities; +import android.media.MediaCodecList; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; +import android.view.Surface; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import org.mozilla.gecko.gfx.GeckoSurface; + +/* package */ final class Codec extends ICodec.Stub implements IBinder.DeathRecipient { + private static final String LOGTAG = "GeckoRemoteCodec"; + private static final boolean DEBUG = false; + public static final String SW_CODEC_PREFIX = "OMX.google."; + + public enum Error { + DECODE, + FATAL + } + + private final class Callbacks implements AsyncCodec.Callbacks { + @Override + public void onInputBufferAvailable(final AsyncCodec codec, final int index) { + mInputProcessor.onBuffer(index); + } + + @Override + public void onOutputBufferAvailable( + final AsyncCodec codec, final int index, final MediaCodec.BufferInfo info) { + mOutputProcessor.onBuffer(index, info); + } + + @Override + public void onError(final AsyncCodec codec, final int error) { + reportError(Error.FATAL, new Exception("codec error:" + error)); + } + + @Override + public void onOutputFormatChanged(final AsyncCodec codec, final MediaFormat format) { + mOutputProcessor.onFormatChanged(format); + } + } + + private static final class Input { + public final Sample sample; + public boolean reported; + + public Input(final Sample sample) { + this.sample = sample; + } + } + + private final class InputProcessor { + private boolean mHasInputCapacitySet; + private Queue mAvailableInputBuffers = new LinkedList<>(); + private Queue mDequeuedSamples = new LinkedList<>(); + private Queue mInputSamples = new LinkedList<>(); + private boolean mStopped; + + private synchronized Sample onAllocate(final int size) { + final Sample sample = mSamplePool.obtainInput(size); + sample.session = mSession; + mDequeuedSamples.add(sample); + return sample; + } + + private synchronized void onSample(final Sample sample) { + if (sample == null) { + // Ignore empty input. + mSamplePool.recycleInput(mDequeuedSamples.remove()); + Log.w(LOGTAG, "WARN: empty input sample"); + return; + } + + if (sample.isEOS()) { + queueSample(sample); + return; + } + + if (sample.session >= mSession) { + final Sample dequeued = mDequeuedSamples.remove(); + dequeued.setBufferInfo(sample.info); + dequeued.setCryptoInfo(sample.cryptoInfo); + queueSample(dequeued); + } + + sample.dispose(); + } + + private void queueSample(final Sample sample) { + if (!mInputSamples.offer(new Input(sample))) { + reportError(Error.FATAL, new Exception("FAIL: input sample queue is full")); + return; + } + + try { + feedSampleToBuffer(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + private synchronized void onBuffer(final int index) { + if (mStopped || !isValidBuffer(index)) { + return; + } + + if (!mHasInputCapacitySet) { + final int capacity = mCodec.getInputBuffer(index).capacity(); + if (capacity > 0) { + mSamplePool.setInputBufferSize(capacity); + mHasInputCapacitySet = true; + } + } + + if (mAvailableInputBuffers.offer(index)) { + feedSampleToBuffer(); + } else { + reportError(Error.FATAL, new Exception("FAIL: input buffer queue is full")); + } + } + + private boolean isValidBuffer(final int index) { + try { + return mCodec.getInputBuffer(index) != null; + } catch (final IllegalStateException e) { + if (DEBUG) { + Log.d(LOGTAG, "invalid input buffer#" + index, e); + } + return false; + } + } + + private void feedSampleToBuffer() { + while (!mAvailableInputBuffers.isEmpty() && !mInputSamples.isEmpty()) { + final int index = mAvailableInputBuffers.poll(); + if (!isValidBuffer(index)) { + continue; + } + int len = 0; + final Sample sample = mInputSamples.poll().sample; + final long pts = sample.info.presentationTimeUs; + final int flags = sample.info.flags; + final MediaCodec.CryptoInfo cryptoInfo = sample.cryptoInfo; + if (!sample.isEOS() && sample.bufferId != Sample.NO_BUFFER) { + len = sample.info.size; + final ByteBuffer buf = mCodec.getInputBuffer(index); + try { + mSamplePool + .getInputBuffer(sample.bufferId) + .writeToByteBuffer(buf, sample.info.offset, len); + } catch (final IOException e) { + e.printStackTrace(); + len = 0; + } + mSamplePool.recycleInput(sample); + } + + try { + if (cryptoInfo != null && len > 0) { + mCodec.queueSecureInputBuffer(index, 0, cryptoInfo, pts, flags); + } else { + mCodec.queueInputBuffer(index, 0, len, pts, flags); + } + mCallbacks.onInputQueued(pts); + } catch (final RemoteException e) { + e.printStackTrace(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + return; + } + } + reportPendingInputs(); + } + + private void reportPendingInputs() { + try { + for (final Input i : mInputSamples) { + if (!i.reported) { + i.reported = true; + mCallbacks.onInputPending(i.sample.info.presentationTimeUs); + } + } + } catch (final RemoteException e) { + e.printStackTrace(); + } + } + + private synchronized void reset() { + for (final Input i : mInputSamples) { + if (!i.sample.isEOS()) { + mSamplePool.recycleInput(i.sample); + } + } + mInputSamples.clear(); + + for (final Sample s : mDequeuedSamples) { + mSamplePool.recycleInput(s); + } + mDequeuedSamples.clear(); + + mAvailableInputBuffers.clear(); + } + + private synchronized void start() { + if (!mStopped) { + return; + } + mStopped = false; + } + + private synchronized void stop() { + if (mStopped) { + return; + } + mStopped = true; + reset(); + } + } + + private static final class Output { + public final Sample sample; + public final int index; + + public Output(final Sample sample, final int index) { + this.sample = sample; + this.index = index; + } + } + + private class OutputProcessor { + private final boolean mRenderToSurface; + private boolean mHasOutputCapacitySet; + private Queue mSentOutputs = new LinkedList<>(); + private boolean mStopped; + + private OutputProcessor(final boolean renderToSurface) { + mRenderToSurface = renderToSurface; + } + + private synchronized void onBuffer(final int index, final MediaCodec.BufferInfo info) { + if (mStopped || !isValidBuffer(index)) { + return; + } + + try { + final Sample output = obtainOutputSample(index, info); + mSentOutputs.add(new Output(output, index)); + output.session = mSession; + mCallbacks.onOutput(output); + } catch (final Exception e) { + e.printStackTrace(); + mCodec.releaseOutputBuffer(index, false); + } + + final boolean eos = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + if (DEBUG && eos) { + Log.d(LOGTAG, "output EOS"); + } + } + + private boolean isValidBuffer(final int index) { + try { + return (mCodec.getOutputBuffer(index) != null) || mRenderToSurface; + } catch (final IllegalStateException e) { + if (DEBUG) { + Log.e(LOGTAG, "invalid buffer#" + index, e); + } + return false; + } + } + + private Sample obtainOutputSample(final int index, final MediaCodec.BufferInfo info) { + final Sample sample = mSamplePool.obtainOutput(info); + + if (mRenderToSurface) { + return sample; + } + + final ByteBuffer output = mCodec.getOutputBuffer(index); + if (!mHasOutputCapacitySet) { + final int capacity = output.capacity(); + if (capacity > 0) { + mSamplePool.setOutputBufferSize(capacity); + mHasOutputCapacitySet = true; + } + } + + if (info.size > 0) { + try { + mSamplePool + .getOutputBuffer(sample.bufferId) + .readFromByteBuffer(output, info.offset, info.size); + } catch (final IOException e) { + Log.e(LOGTAG, "Fail to read output buffer:" + e.getMessage()); + } + } + + return sample; + } + + private synchronized void onRelease(final Sample sample, final boolean render) { + final Output output = mSentOutputs.poll(); + if (output != null) { + mCodec.releaseOutputBuffer(output.index, render); + mSamplePool.recycleOutput(output.sample); + } else if (DEBUG) { + Log.d(LOGTAG, sample + " already released"); + } + + sample.dispose(); + } + + private synchronized void onFormatChanged(final MediaFormat format) { + if (mStopped) { + return; + } + try { + mCallbacks.onOutputFormatChanged(new FormatParam(format)); + } catch (final RemoteException re) { + // Dead recipient. + re.printStackTrace(); + } + } + + private synchronized void reset() { + for (final Output o : mSentOutputs) { + mCodec.releaseOutputBuffer(o.index, false); + mSamplePool.recycleOutput(o.sample); + } + mSentOutputs.clear(); + } + + private synchronized void start() { + if (!mStopped) { + return; + } + mStopped = false; + } + + private synchronized void stop() { + if (mStopped) { + return; + } + mStopped = true; + reset(); + } + } + + private volatile ICodecCallbacks mCallbacks; + private GeckoSurface mSurface; + private AsyncCodec mCodec; + private InputProcessor mInputProcessor; + private OutputProcessor mOutputProcessor; + private long mSession; + private SamplePool mSamplePool; + // Values will be updated after configure called. + private volatile boolean mIsAdaptivePlaybackSupported = false; + private volatile boolean mIsHardwareAccelerated = false; + private boolean mIsTunneledPlaybackSupported = false; + + public synchronized void setCallbacks(final ICodecCallbacks callbacks) throws RemoteException { + mCallbacks = callbacks; + callbacks.asBinder().linkToDeath(this, 0); + } + + // IBinder.DeathRecipient + @Override + public synchronized void binderDied() { + Log.e(LOGTAG, "Callbacks is dead"); + try { + release(); + } catch (final RemoteException e) { + // Nowhere to report the error. + } + } + + @Override + public synchronized boolean configure( + final FormatParam format, final GeckoSurface surface, final int flags, final String drmStubId) + throws RemoteException { + if (mCallbacks == null) { + Log.e(LOGTAG, "FAIL: callbacks must be set before calling configure()"); + return false; + } + + if (mCodec != null) { + if (DEBUG) { + Log.d(LOGTAG, "release existing codec: " + mCodec); + } + mCodec.release(); + } + + if (DEBUG) { + Log.d(LOGTAG, "configure " + this); + } + + final MediaFormat fmt = format.asFormat(); + final String mime = fmt.getString(MediaFormat.KEY_MIME); + if (mime == null || mime.isEmpty()) { + Log.e(LOGTAG, "invalid MIME type: " + mime); + return false; + } + + final List found = + findMatchingCodecNames(fmt, flags == MediaCodec.CONFIGURE_FLAG_ENCODE); + for (final String name : found) { + final AsyncCodec codec = + configureCodec( + name, fmt, surface != null ? surface.getSurface() : null, flags, drmStubId); + if (codec == null) { + Log.w(LOGTAG, "unable to configure " + name + ". Try next."); + continue; + } + mIsHardwareAccelerated = !name.startsWith(SW_CODEC_PREFIX); + mCodec = codec; + // Bug 1789846: Check if the Codec provides stride or height values to use. + if (flags == MediaCodec.CONFIGURE_FLAG_ENCODE && fmt.containsKey(MediaFormat.KEY_WIDTH)) { + final MediaFormat inputFormat = mCodec.getInputFormat(); + if (inputFormat != null) { + if (inputFormat.containsKey(MediaFormat.KEY_STRIDE)) { + fmt.setInteger(MediaFormat.KEY_STRIDE, inputFormat.getInteger(MediaFormat.KEY_STRIDE)); + } + if (inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + fmt.setInteger( + MediaFormat.KEY_SLICE_HEIGHT, inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT)); + } + } + } + mInputProcessor = new InputProcessor(); + final boolean renderToSurface = surface != null; + mOutputProcessor = new OutputProcessor(renderToSurface); + mSamplePool = new SamplePool(name, renderToSurface); + if (renderToSurface) { + mIsTunneledPlaybackSupported = mCodec.isTunneledPlaybackSupported(mime); + mSurface = surface; // Take ownership of surface. + } + if (DEBUG) { + Log.d(LOGTAG, codec.toString() + " created. Render to surface?" + renderToSurface); + } + return true; + } + + return false; + } + + private List findMatchingCodecNames(final MediaFormat format, final boolean isEncoder) { + final String mimeType = format.getString(MediaFormat.KEY_MIME); + // Missing width and height value in format means audio; + // Video format should never has 0 width or height. + final int width = + format.containsKey(MediaFormat.KEY_WIDTH) ? format.getInteger(MediaFormat.KEY_WIDTH) : 0; + final int height = + format.containsKey(MediaFormat.KEY_HEIGHT) ? format.getInteger(MediaFormat.KEY_HEIGHT) : 0; + + int numCodecs = 0; + final List found = new ArrayList<>(); + try { + numCodecs = MediaCodecList.getCodecCount(); + } catch (final RuntimeException e) { + Log.e(LOGTAG, "Failed retrieving codec count finding matching codec names", e); + return found; + } + + for (int i = 0; i < numCodecs; i++) { + final MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); + if (info.isEncoder() == !isEncoder) { + continue; + } + + final String[] types = info.getSupportedTypes(); + for (final String t : types) { + if (!t.equalsIgnoreCase(mimeType)) { + continue; + } + final String name = info.getName(); + // API 21+ provide a method to query whether supplied size is supported. For + // older version, just avoid software video encoders. + if (isEncoder && width > 0 && height > 0) { + final VideoCapabilities c = info.getCapabilitiesForType(mimeType).getVideoCapabilities(); + if (c != null && !c.isSizeSupported(width, height)) { + if (DEBUG) { + Log.d(LOGTAG, name + ": " + width + "x" + height + " not supported"); + } + continue; + } + } + + found.add(name); + if (DEBUG) { + Log.d( + LOGTAG, + "found " + (isEncoder ? "encoder:" : "decoder:") + name + " for mime:" + mimeType); + } + } + } + return found; + } + + private AsyncCodec configureCodec( + final String name, + final MediaFormat format, + final Surface surface, + final int flags, + final String drmStubId) { + try { + final AsyncCodec codec = AsyncCodecFactory.create(name); + codec.setCallbacks(new Callbacks(), null); + + final MediaCrypto crypto = RemoteMediaDrmBridgeStub.getMediaCrypto(drmStubId); + if (DEBUG) { + Log.d( + LOGTAG, + "configure mediacodec with crypto(" + (crypto != null) + ") / Id :" + drmStubId); + } + + if (surface != null) { + setupAdaptivePlayback(codec, format); + } + + codec.configure(format, surface, crypto, flags); + return codec; + } catch (final Exception e) { + Log.e(LOGTAG, "codec creation error", e); + return null; + } + } + + private void setupAdaptivePlayback(final AsyncCodec codec, final MediaFormat format) { + // Video decoder should config with adaptive playback capability. + mIsAdaptivePlaybackSupported = + codec.isAdaptivePlaybackSupported(format.getString(MediaFormat.KEY_MIME)); + if (mIsAdaptivePlaybackSupported) { + if (DEBUG) { + Log.d(LOGTAG, "codec supports adaptive playback = " + mIsAdaptivePlaybackSupported); + } + // TODO: may need to find a way to not use hard code to decide the max w/h. + format.setInteger(MediaFormat.KEY_MAX_WIDTH, 1920); + format.setInteger(MediaFormat.KEY_MAX_HEIGHT, 1080); + } + } + + @Override + public synchronized boolean isAdaptivePlaybackSupported() { + return mIsAdaptivePlaybackSupported; + } + + @Override + public synchronized boolean isHardwareAccelerated() { + return mIsHardwareAccelerated; + } + + @Override + public synchronized boolean isTunneledPlaybackSupported() { + return mIsTunneledPlaybackSupported; + } + + @Override + public synchronized void start() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "start " + this); + } + mInputProcessor.start(); + mOutputProcessor.start(); + try { + mCodec.start(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + private void reportError(final Error error, final Exception e) { + if (e != null) { + e.printStackTrace(); + } + try { + mCallbacks.onError(error == Error.FATAL); + } catch (final NullPointerException ne) { + // mCallbacks has been disposed by release(). + } catch (final RemoteException re) { + re.printStackTrace(); + } + } + + @Override + public synchronized void stop() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "stop " + this); + } + try { + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.stop(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void flush() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "flush " + this); + } + try { + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.flush(); + if (DEBUG) { + Log.d(LOGTAG, "flushed " + this); + } + mInputProcessor.start(); + mOutputProcessor.start(); + mCodec.resumeReceivingInputs(); + mSession++; + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized Sample dequeueInput(final int size) throws RemoteException { + try { + return mInputProcessor.onAllocate(size); + } catch (final Exception e) { + // Translate allocation error to remote exception. + throw new RemoteException(e.getMessage()); + } + } + + @Override + public synchronized SampleBuffer getInputBuffer(final int id) { + if (mSamplePool == null) { + return null; + } + return mSamplePool.getInputBuffer(id); + } + + @Override + public synchronized SampleBuffer getOutputBuffer(final int id) { + if (mSamplePool == null) { + return null; + } + return mSamplePool.getOutputBuffer(id); + } + + @Override + public synchronized void queueInput(final Sample sample) throws RemoteException { + try { + mInputProcessor.onSample(sample); + } catch (final Exception e) { + throw new RemoteException(e.getMessage()); + } + } + + @Override + public synchronized void setBitrate(final int bps) { + try { + mCodec.setBitrate(bps); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void releaseOutput(final Sample sample, final boolean render) { + try { + mOutputProcessor.onRelease(sample, render); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void release() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "release " + this); + } + try { + // In case Codec.stop() is not called yet. + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.release(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + mCodec = null; + mSamplePool.reset(); + mSamplePool = null; + mCallbacks.asBinder().unlinkToDeath(this, 0); + mCallbacks = null; + if (mSurface != null) { + mSurface.release(); + mSurface = null; + } + } +} -- cgit v1.2.3