1 files changed, 712 insertions, 0 deletions
diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java
new file mode 100644
index 0000000000..dc9d9e3862
--- /dev/null
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java
@@ -0,0 +1,712 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.media;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaCodecInfo.VideoCapabilities;
+import android.media.MediaCodecList;
+import android.media.MediaCrypto;
+import android.media.MediaFormat;
+import android.os.Build;
+import android.os.IBinder;
+import android.os.RemoteException;
+import android.util.Log;
+import android.view.Surface;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import org.mozilla.gecko.gfx.GeckoSurface;
+
+/* package */ final class Codec extends ICodec.Stub implements IBinder.DeathRecipient {
+  private static final String LOGTAG = "GeckoRemoteCodec";
+  private static final boolean DEBUG = false;
+  public static final String SW_CODEC_PREFIX = "OMX.google.";
+
+  public enum Error {
+    DECODE,
+    FATAL
+  }
+
+  private final class Callbacks implements AsyncCodec.Callbacks {
+    @Override
+    public void onInputBufferAvailable(final AsyncCodec codec, final int index) {
+      mInputProcessor.onBuffer(index);
+    }
+
+    @Override
+    public void onOutputBufferAvailable(
+        final AsyncCodec codec, final int index, final MediaCodec.BufferInfo info) {
+      mOutputProcessor.onBuffer(index, info);
+    }
+
+    @Override
+    public void onError(final AsyncCodec codec, final int error) {
+      reportError(Error.FATAL, new Exception("codec error:" + error));
+    }
+
+    @Override
+    public void onOutputFormatChanged(final AsyncCodec codec, final MediaFormat format) {
+      mOutputProcessor.onFormatChanged(format);
+    }
+  }
+
+  private static final class Input {
+    public final Sample sample;
+    public boolean reported;
+
+    public Input(final Sample sample) {
+      this.sample = sample;
+    }
+  }
+
+  private final class InputProcessor {
+    private boolean mHasInputCapacitySet;
+    private Queue<Integer> mAvailableInputBuffers = new LinkedList<>();
+    private Queue<Sample> mDequeuedSamples = new LinkedList<>();
+    private Queue<Input> mInputSamples = new LinkedList<>();
+    private boolean mStopped;
+
+    private synchronized Sample onAllocate(final int size) {
+      final Sample sample = mSamplePool.obtainInput(size);
+      sample.session = mSession;
+      mDequeuedSamples.add(sample);
+      return sample;
+    }
+
+    private synchronized void onSample(final Sample sample) {
+      if (sample == null) {
+        // Ignore empty input.
+        mSamplePool.recycleInput(mDequeuedSamples.remove());
+        Log.w(LOGTAG, "WARN: empty input sample");
+        return;
+      }
+
+      if (sample.isEOS()) {
+        queueSample(sample);
+        return;
+      }
+
+      if (sample.session >= mSession) {
+        final Sample dequeued = mDequeuedSamples.remove();
+        dequeued.setBufferInfo(sample.info);
+        dequeued.setCryptoInfo(sample.cryptoInfo);
+        queueSample(dequeued);
+      }
+
+      sample.dispose();
+    }
+
+    private void queueSample(final Sample sample) {
+      if (!mInputSamples.offer(new Input(sample))) {
+        reportError(Error.FATAL, new Exception("FAIL: input sample queue is full"));
+        return;
+      }
+
+      try {
+        feedSampleToBuffer();
+      } catch (final Exception e) {
+        reportError(Error.FATAL, e);
+      }
+    }
+
+    private synchronized void onBuffer(final int index) {
+      if (mStopped || !isValidBuffer(index)) {
+        return;
+      }
+
+      if (!mHasInputCapacitySet) {
+        final int capacity = mCodec.getInputBuffer(index).capacity();
+        if (capacity > 0) {
+          mSamplePool.setInputBufferSize(capacity);
+          mHasInputCapacitySet = true;
+        }
+      }
+
+      if (mAvailableInputBuffers.offer(index)) {
+        feedSampleToBuffer();
+      } else {
+        reportError(Error.FATAL, new Exception("FAIL: input buffer queue is full"));
+      }
+    }
+
+    private boolean isValidBuffer(final int index) {
+      try {
+        return mCodec.getInputBuffer(index) != null;
+      } catch (final IllegalStateException e) {
+        if (DEBUG) {
+          Log.d(LOGTAG, "invalid input buffer#" + index, e);
+        }
+        return false;
+      }
+    }
+
+    private void feedSampleToBuffer() {
+      while (!mAvailableInputBuffers.isEmpty() && !mInputSamples.isEmpty()) {
+        final int index = mAvailableInputBuffers.poll();
+        if (!isValidBuffer(index)) {
+          continue;
+        }
+        int len = 0;
+        final Sample sample = mInputSamples.poll().sample;
+        final long pts = sample.info.presentationTimeUs;
+        final int flags = sample.info.flags;
+        final MediaCodec.CryptoInfo cryptoInfo = sample.cryptoInfo;
+        if (!sample.isEOS() && sample.bufferId != Sample.NO_BUFFER) {
+          len = sample.info.size;
+          final ByteBuffer buf = mCodec.getInputBuffer(index);
+          try {
+            mSamplePool
+                .getInputBuffer(sample.bufferId)
+                .writeToByteBuffer(buf, sample.info.offset, len);
+          } catch (final IOException e) {
+            e.printStackTrace();
+            len = 0;
+          }
+          mSamplePool.recycleInput(sample);
+        }
+
+        try {
+          if (cryptoInfo != null && len > 0) {
+            mCodec.queueSecureInputBuffer(index, 0, cryptoInfo, pts, flags);
+          } else {
+            mCodec.queueInputBuffer(index, 0, len, pts, flags);
+          }
+          mCallbacks.onInputQueued(pts);
+        } catch (final RemoteException e) {
+          e.printStackTrace();
+        } catch (final Exception e) {
+          reportError(Error.FATAL, e);
+          return;
+        }
+      }
+      reportPendingInputs();
+    }
+
+    private void reportPendingInputs() {
+      try {
+        for (final Input i : mInputSamples) {
+          if (!i.reported) {
+            i.reported = true;
+            mCallbacks.onInputPending(i.sample.info.presentationTimeUs);
+          }
+        }
+      } catch (final RemoteException e) {
+        e.printStackTrace();
+      }
+    }
+
+    private synchronized void reset() {
+      for (final Input i : mInputSamples) {
+        if (!i.sample.isEOS()) {
+          mSamplePool.recycleInput(i.sample);
+        }
+      }
+      mInputSamples.clear();
+
+      for (final Sample s : mDequeuedSamples) {
+        mSamplePool.recycleInput(s);
+      }
+      mDequeuedSamples.clear();
+
+      mAvailableInputBuffers.clear();
+    }
+
+    private synchronized void start() {
+      if (!mStopped) {
+        return;
+      }
+      mStopped = false;
+    }
+
+    private synchronized void stop() {
+      if (mStopped) {
+        return;
+      }
+      mStopped = true;
+      reset();
+    }
+  }
+
+  private static final class Output {
+    public final Sample sample;
+    public final int index;
+
+    public Output(final Sample sample, final int index) {
+      this.sample = sample;
+      this.index = index;
+    }
+  }
+
+  private class OutputProcessor {
+    private final boolean mRenderToSurface;
+    private boolean mHasOutputCapacitySet;
+    private Queue<Output> mSentOutputs = new LinkedList<>();
+    private boolean mStopped;
+
+    private OutputProcessor(final boolean renderToSurface) {
+      mRenderToSurface = renderToSurface;
+    }
+
+    private synchronized void onBuffer(final int index, final MediaCodec.BufferInfo info) {
+      if (mStopped || !isValidBuffer(index)) {
+        return;
+      }
+
+      try {
+        final Sample output = obtainOutputSample(index, info);
+        mSentOutputs.add(new Output(output, index));
+        output.session = mSession;
+        mCallbacks.onOutput(output);
+      } catch (final Exception e) {
+        e.printStackTrace();
+        mCodec.releaseOutputBuffer(index, false);
+      }
+
+      final boolean eos = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
+      if (DEBUG && eos) {
+        Log.d(LOGTAG, "output EOS");
+      }
+    }
+
+    private boolean isValidBuffer(final int index) {
+      try {
+        return (mCodec.getOutputBuffer(index) != null) || mRenderToSurface;
+      } catch (final IllegalStateException e) {
+        if (DEBUG) {
+          Log.e(LOGTAG, "invalid buffer#" + index, e);
+        }
+        return false;
+      }
+    }
+
+    private Sample obtainOutputSample(final int index, final MediaCodec.BufferInfo info) {
+      final Sample sample = mSamplePool.obtainOutput(info);
+
+      if (mRenderToSurface) {
+        return sample;
+      }
+
+      final ByteBuffer output = mCodec.getOutputBuffer(index);
+      if (!mHasOutputCapacitySet) {
+        final int capacity = output.capacity();
+        if (capacity > 0) {
+          mSamplePool.setOutputBufferSize(capacity);
+          mHasOutputCapacitySet = true;
+        }
+      }
+
+      if (info.size > 0) {
+        try {
+          mSamplePool
+              .getOutputBuffer(sample.bufferId)
+              .readFromByteBuffer(output, info.offset, info.size);
+        } catch (final IOException e) {
+          Log.e(LOGTAG, "Fail to read output buffer:" + e.getMessage());
+        }
+      }
+
+      return sample;
+    }
+
+    private synchronized void onRelease(final Sample sample, final boolean render) {
+      final Output output = mSentOutputs.poll();
+      if (output != null) {
+        mCodec.releaseOutputBuffer(output.index, render);
+        mSamplePool.recycleOutput(output.sample);
+      } else if (DEBUG) {
+        Log.d(LOGTAG, sample + " already released");
+      }
+
+      sample.dispose();
+    }
+
+    private synchronized void onFormatChanged(final MediaFormat format) {
+      if (mStopped) {
+        return;
+      }
+      try {
+        mCallbacks.onOutputFormatChanged(new FormatParam(format));
+      } catch (final RemoteException re) {
+        // Dead recipient.
+        re.printStackTrace();
+      }
+    }
+
+    private synchronized void reset() {
+      for (final Output o : mSentOutputs) {
+        mCodec.releaseOutputBuffer(o.index, false);
+        mSamplePool.recycleOutput(o.sample);
+      }
+      mSentOutputs.clear();
+    }
+
+    private synchronized void start() {
+      if (!mStopped) {
+        return;
+      }
+      mStopped = false;
+    }
+
+    private synchronized void stop() {
+      if (mStopped) {
+        return;
+      }
+      mStopped = true;
+      reset();
+    }
+  }
+
+  private volatile ICodecCallbacks mCallbacks;
+  private GeckoSurface mSurface;
+  private AsyncCodec mCodec;
+  private InputProcessor mInputProcessor;
+  private OutputProcessor mOutputProcessor;
+  private long mSession;
+  private SamplePool mSamplePool;
+  // Values will be updated after configure called.
+  private volatile boolean mIsAdaptivePlaybackSupported = false;
+  private volatile boolean mIsHardwareAccelerated = false;
+  private boolean mIsTunneledPlaybackSupported = false;
+
+  public synchronized void setCallbacks(final ICodecCallbacks callbacks) throws RemoteException {
+    mCallbacks = callbacks;
+    callbacks.asBinder().linkToDeath(this, 0);
+  }
+
+  // IBinder.DeathRecipient
+  @Override
+  public synchronized void binderDied() {
+    Log.e(LOGTAG, "Callbacks is dead");
+    try {
+      release();
+    } catch (final RemoteException e) {
+      // Nowhere to report the error.
+    }
+  }
+
+  @Override
+  public synchronized boolean configure(
+      final FormatParam format, final GeckoSurface surface, final int flags, final String drmStubId)
+      throws RemoteException {
+    if (mCallbacks == null) {
+      Log.e(LOGTAG, "FAIL: callbacks must be set before calling configure()");
+      return false;
+    }
+
+    if (mCodec != null) {
+      if (DEBUG) {
+        Log.d(LOGTAG, "release existing codec: " + mCodec);
+      }
+      mCodec.release();
+    }
+
+    if (DEBUG) {
+      Log.d(LOGTAG, "configure " + this);
+    }
+
+    final MediaFormat fmt = format.asFormat();
+    final String mime = fmt.getString(MediaFormat.KEY_MIME);
+    if (mime == null || mime.isEmpty()) {
+      Log.e(LOGTAG, "invalid MIME type: " + mime);
+      return false;
+    }
+
+    final List<String> found =
+        findMatchingCodecNames(fmt, flags == MediaCodec.CONFIGURE_FLAG_ENCODE);
+    for (final String name : found) {
+      final AsyncCodec codec =
+          configureCodec(
+              name, fmt, surface != null ? surface.getSurface() : null, flags, drmStubId);
+      if (codec == null) {
+        Log.w(LOGTAG, "unable to configure " + name + ". Try next.");
+        continue;
+      }
+      mIsHardwareAccelerated = !name.startsWith(SW_CODEC_PREFIX);
+      mCodec = codec;
+      // Bug 1789846: Check if the Codec provides stride or height values to use.
+      if (flags == MediaCodec.CONFIGURE_FLAG_ENCODE && fmt.containsKey(MediaFormat.KEY_WIDTH)) {
+        final MediaFormat inputFormat = mCodec.getInputFormat();
+        if (inputFormat != null) {
+          if (inputFormat.containsKey(MediaFormat.KEY_STRIDE)) {
+            fmt.setInteger(MediaFormat.KEY_STRIDE, inputFormat.getInteger(MediaFormat.KEY_STRIDE));
+          }
+          if (inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+            fmt.setInteger(
+                MediaFormat.KEY_SLICE_HEIGHT, inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT));
+          }
+        }
+      }
+      mInputProcessor = new InputProcessor();
+      final boolean renderToSurface = surface != null;
+      mOutputProcessor = new OutputProcessor(renderToSurface);
+      mSamplePool = new SamplePool(name, renderToSurface);
+      if (renderToSurface) {
+        mIsTunneledPlaybackSupported = mCodec.isTunneledPlaybackSupported(mime);
+        mSurface = surface; // Take ownership of surface.
+      }
+      if (DEBUG) {
+        Log.d(LOGTAG, codec.toString() + " created. Render to surface?" + renderToSurface);
+      }
+      return true;
+    }
+
+    return false;
+  }
+
+  private List<String> findMatchingCodecNames(final MediaFormat format, final boolean isEncoder) {
+    final String mimeType = format.getString(MediaFormat.KEY_MIME);
+    // Missing width and height value in format means audio;
+    // Video format should never has 0 width or height.
+    final int width =
+        format.containsKey(MediaFormat.KEY_WIDTH) ? format.getInteger(MediaFormat.KEY_WIDTH) : 0;
+    final int height =
+        format.containsKey(MediaFormat.KEY_HEIGHT) ? format.getInteger(MediaFormat.KEY_HEIGHT) : 0;
+
+    final int numCodecs = MediaCodecList.getCodecCount();
+    final List<String> found = new ArrayList<>();
+    for (int i = 0; i < numCodecs; i++) {
+      final MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
+      if (info.isEncoder() == !isEncoder) {
+        continue;
+      }
+
+      final String[] types = info.getSupportedTypes();
+      for (final String t : types) {
+        if (!t.equalsIgnoreCase(mimeType)) {
+          continue;
+        }
+        final String name = info.getName();
+        // API 21+ provide a method to query whether supplied size is supported. For
+        // older version, just avoid software video encoders.
+        if (isEncoder && width > 0 && height > 0) {
+          if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            final VideoCapabilities c =
+                info.getCapabilitiesForType(mimeType).getVideoCapabilities();
+            if (c != null && !c.isSizeSupported(width, height)) {
+              if (DEBUG) {
+                Log.d(LOGTAG, name + ": " + width + "x" + height + " not supported");
+              }
+              continue;
+            }
+          } else if (name.startsWith(SW_CODEC_PREFIX)) {
+            continue;
+          }
+        }
+
+        found.add(name);
+        if (DEBUG) {
+          Log.d(
+              LOGTAG,
+              "found " + (isEncoder ? "encoder:" : "decoder:") + name + " for mime:" + mimeType);
+        }
+      }
+    }
+    return found;
+  }
+
+  private AsyncCodec configureCodec(
+      final String name,
+      final MediaFormat format,
+      final Surface surface,
+      final int flags,
+      final String drmStubId) {
+    try {
+      final AsyncCodec codec = AsyncCodecFactory.create(name);
+      codec.setCallbacks(new Callbacks(), null);
+
+      final MediaCrypto crypto = RemoteMediaDrmBridgeStub.getMediaCrypto(drmStubId);
+      if (DEBUG) {
+        Log.d(
+            LOGTAG,
+            "configure mediacodec with crypto(" + (crypto != null) + ") / Id :" + drmStubId);
+      }
+
+      if (surface != null) {
+        setupAdaptivePlayback(codec, format);
+      }
+
+      codec.configure(format, surface, crypto, flags);
+      return codec;
+    } catch (final Exception e) {
+      Log.e(LOGTAG, "codec creation error", e);
+      return null;
+    }
+  }
+
+  private void setupAdaptivePlayback(final AsyncCodec codec, final MediaFormat format) {
+    // Video decoder should config with adaptive playback capability.
+    mIsAdaptivePlaybackSupported =
+        codec.isAdaptivePlaybackSupported(format.getString(MediaFormat.KEY_MIME));
+    if (mIsAdaptivePlaybackSupported) {
+      if (DEBUG) {
+        Log.d(LOGTAG, "codec supports adaptive playback  = " + mIsAdaptivePlaybackSupported);
+      }
+      // TODO: may need to find a way to not use hard code to decide the max w/h.
+      format.setInteger(MediaFormat.KEY_MAX_WIDTH, 1920);
+      format.setInteger(MediaFormat.KEY_MAX_HEIGHT, 1080);
+    }
+  }
+
+  @Override
+  public synchronized boolean isAdaptivePlaybackSupported() {
+    return mIsAdaptivePlaybackSupported;
+  }
+
+  @Override
+  public synchronized boolean isHardwareAccelerated() {
+    return mIsHardwareAccelerated;
+  }
+
+  @Override
+  public synchronized boolean isTunneledPlaybackSupported() {
+    return mIsTunneledPlaybackSupported;
+  }
+
+  @Override
+  public synchronized void start() throws RemoteException {
+    if (DEBUG) {
+      Log.d(LOGTAG, "start " + this);
+    }
+    mInputProcessor.start();
+    mOutputProcessor.start();
+    try {
+      mCodec.start();
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+  }
+
+  private void reportError(final Error error, final Exception e) {
+    if (e != null) {
+      e.printStackTrace();
+    }
+    try {
+      mCallbacks.onError(error == Error.FATAL);
+    } catch (final NullPointerException ne) {
+      // mCallbacks has been disposed by release().
+    } catch (final RemoteException re) {
+      re.printStackTrace();
+    }
+  }
+
+  @Override
+  public synchronized void stop() throws RemoteException {
+    if (DEBUG) {
+      Log.d(LOGTAG, "stop " + this);
+    }
+    try {
+      mInputProcessor.stop();
+      mOutputProcessor.stop();
+
+      mCodec.stop();
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+  }
+
+  @Override
+  public synchronized void flush() throws RemoteException {
+    if (DEBUG) {
+      Log.d(LOGTAG, "flush " + this);
+    }
+    try {
+      mInputProcessor.stop();
+      mOutputProcessor.stop();
+
+      mCodec.flush();
+      if (DEBUG) {
+        Log.d(LOGTAG, "flushed " + this);
+      }
+      mInputProcessor.start();
+      mOutputProcessor.start();
+      mCodec.resumeReceivingInputs();
+      mSession++;
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+  }
+
+  @Override
+  public synchronized Sample dequeueInput(final int size) throws RemoteException {
+    try {
+      return mInputProcessor.onAllocate(size);
+    } catch (final Exception e) {
+      // Translate allocation error to remote exception.
+      throw new RemoteException(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized SampleBuffer getInputBuffer(final int id) {
+    if (mSamplePool == null) {
+      return null;
+    }
+    return mSamplePool.getInputBuffer(id);
+  }
+
+  @Override
+  public synchronized SampleBuffer getOutputBuffer(final int id) {
+    if (mSamplePool == null) {
+      return null;
+    }
+    return mSamplePool.getOutputBuffer(id);
+  }
+
+  @Override
+  public synchronized void queueInput(final Sample sample) throws RemoteException {
+    try {
+      mInputProcessor.onSample(sample);
+    } catch (final Exception e) {
+      throw new RemoteException(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized void setBitrate(final int bps) {
+    try {
+      mCodec.setBitrate(bps);
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+  }
+
+  @Override
+  public synchronized void releaseOutput(final Sample sample, final boolean render) {
+    try {
+      mOutputProcessor.onRelease(sample, render);
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+  }
+
+  @Override
+  public synchronized void release() throws RemoteException {
+    if (DEBUG) {
+      Log.d(LOGTAG, "release " + this);
+    }
+    try {
+      // In case Codec.stop() is not called yet.
+      mInputProcessor.stop();
+      mOutputProcessor.stop();
+
+      mCodec.release();
+    } catch (final Exception e) {
+      reportError(Error.FATAL, e);
+    }
+    mCodec = null;
+    mSamplePool.reset();
+    mSamplePool = null;
+    mCallbacks.asBinder().unlinkToDeath(this, 0);
+    mCallbacks = null;
+    if (mSurface != null) {
+      mSurface.release();
+      mSurface = null;
+    }
+  }
+}