1 files changed, 508 insertions, 0 deletions
diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java
new file mode 100644
index 0000000000..e31ea4b132
--- /dev/null
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java
@@ -0,0 +1,508 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.media;
+
+import android.media.MediaCodec;
+import android.media.MediaCodec.BufferInfo;
+import android.media.MediaCodec.CryptoInfo;
+import android.media.MediaFormat;
+import android.os.Build;
+import android.os.DeadObjectException;
+import android.os.RemoteException;
+import android.util.Log;
+import android.util.SparseArray;
+import androidx.annotation.RequiresApi;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.mozilla.gecko.annotation.WrapForJNI;
+import org.mozilla.gecko.gfx.GeckoSurface;
+import org.mozilla.gecko.mozglue.JNIObject;
+
+// Proxy class of ICodec binder.
+public final class CodecProxy {
+  private static final String LOGTAG = "GeckoRemoteCodecProxy";
+  private static final boolean DEBUG = false;
+  @WrapForJNI private static final long INVALID_SESSION = -1;
+
+  private ICodec mRemote;
+  private long mSession;
+  private boolean mIsEncoder;
+  private FormatParam mFormat;
+  private GeckoSurface mOutputSurface;
+  private CallbacksForwarder mCallbacks;
+  private String mRemoteDrmStubId;
+  private Queue<Sample> mSurfaceOutputs = new ConcurrentLinkedQueue<>();
+  private boolean mFlushed = true;
+
+  private SparseArray<SampleBuffer> mInputBuffers = new SparseArray<>();
+  private SparseArray<SampleBuffer> mOutputBuffers = new SparseArray<>();
+
+  public interface Callbacks {
+    void onInputStatus(long timestamp, boolean processed);
+
+    void onOutputFormatChanged(MediaFormat format);
+
+    void onOutput(Sample output, SampleBuffer buffer);
+
+    void onError(boolean fatal);
+  }
+
+  @WrapForJNI
+  public static class NativeCallbacks extends JNIObject implements Callbacks {
+    public native void onInputStatus(long timestamp, boolean processed);
+
+    public native void onOutputFormatChanged(MediaFormat format);
+
+    public native void onOutput(Sample output, SampleBuffer buffer);
+
+    public native void onError(boolean fatal);
+
+    @Override // JNIObject
+    protected void disposeNative() {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  private class CallbacksForwarder extends ICodecCallbacks.Stub {
+    private final Callbacks mCallbacks;
+    private boolean mCodecProxyReleased;
+
+    CallbacksForwarder(final Callbacks callbacks) {
+      mCallbacks = callbacks;
+    }
+
+    @Override
+    public synchronized void onInputQueued(final long timestamp) throws RemoteException {
+      if (!mCodecProxyReleased) {
+        mCallbacks.onInputStatus(timestamp, true /* processed */);
+      }
+    }
+
+    @Override
+    public synchronized void onInputPending(final long timestamp) throws RemoteException {
+      if (!mCodecProxyReleased) {
+        mCallbacks.onInputStatus(timestamp, false /* processed */);
+      }
+    }
+
+    @Override
+    public synchronized void onOutputFormatChanged(final FormatParam format)
+        throws RemoteException {
+      if (!mCodecProxyReleased) {
+        mCallbacks.onOutputFormatChanged(format.asFormat());
+      }
+    }
+
+    @Override
+    public synchronized void onOutput(final Sample sample) throws RemoteException {
+      if (mCodecProxyReleased) {
+        sample.dispose();
+        return;
+      }
+
+      final SampleBuffer buffer = CodecProxy.this.getOutputBuffer(sample.bufferId);
+      if (mOutputSurface != null) {
+        // Don't render to surface just yet. Callback will make that happen when it's time.
+        mSurfaceOutputs.offer(sample);
+      } else if (buffer == null) {
+        // Buffer with given ID has been flushed.
+        sample.dispose();
+        return;
+      }
+      mCallbacks.onOutput(sample, buffer);
+    }
+
+    @Override
+    public void onError(final boolean fatal) throws RemoteException {
+      reportError(fatal);
+    }
+
+    private synchronized void reportError(final boolean fatal) {
+      if (!mCodecProxyReleased) {
+        mCallbacks.onError(fatal);
+      }
+    }
+
+    private synchronized void setCodecProxyReleased() {
+      mCodecProxyReleased = true;
+    }
+  }
+
+  @WrapForJNI
+  public int GetInputFormatStride() {
+    if (mFormat.asFormat().containsKey(MediaFormat.KEY_STRIDE)) {
+      return mFormat.asFormat().getInteger(MediaFormat.KEY_STRIDE);
+    }
+    return 0;
+  }
+
+  @WrapForJNI
+  public int GetInputFormatYPlaneHeight() {
+    if (mFormat.asFormat().containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+      return mFormat.asFormat().getInteger(MediaFormat.KEY_SLICE_HEIGHT);
+    }
+    return 0;
+  }
+
+  @WrapForJNI
+  public static CodecProxy create(
+      final boolean isEncoder,
+      final MediaFormat format,
+      final GeckoSurface surface,
+      final Callbacks callbacks,
+      final String drmStubId) {
+    return RemoteManager.getInstance()
+        .createCodec(isEncoder, format, surface, callbacks, drmStubId);
+  }
+
+  public static CodecProxy createCodecProxy(
+      final boolean isEncoder,
+      final MediaFormat format,
+      final GeckoSurface surface,
+      final Callbacks callbacks,
+      final String drmStubId) {
+    return new CodecProxy(isEncoder, format, surface, callbacks, drmStubId);
+  }
+
+  private CodecProxy(
+      final boolean isEncoder,
+      final MediaFormat format,
+      final GeckoSurface surface,
+      final Callbacks callbacks,
+      final String drmStubId) {
+    mIsEncoder = isEncoder;
+    mFormat = new FormatParam(format);
+    mOutputSurface = surface;
+    mRemoteDrmStubId = drmStubId;
+    mCallbacks = new CallbacksForwarder(callbacks);
+  }
+
+  boolean init(final ICodec remote) {
+    try {
+      remote.setCallbacks(mCallbacks);
+      if (!remote.configure(
+          mFormat,
+          mOutputSurface,
+          mIsEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0,
+          mRemoteDrmStubId)) {
+        return false;
+      }
+      remote.start();
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+
+    mRemote = remote;
+    return true;
+  }
+
+  boolean deinit() {
+    try {
+      mRemote.stop();
+      mRemote.release();
+      mRemote = null;
+      return true;
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+
+  @WrapForJNI
+  public synchronized boolean isAdaptivePlaybackSupported() {
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot check isAdaptivePlaybackSupported with an ended codec");
+      return false;
+    }
+    try {
+      return mRemote.isAdaptivePlaybackSupported();
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+
+  @WrapForJNI
+  public synchronized boolean isHardwareAccelerated() {
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot check isHardwareAccelerated with an ended codec");
+      return false;
+    }
+    try {
+      return mRemote.isHardwareAccelerated();
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+
+  @WrapForJNI
+  public synchronized boolean isTunneledPlaybackSupported() {
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot check isTunneledPlaybackSupported with an ended codec");
+      return false;
+    }
+    try {
+      return mRemote.isTunneledPlaybackSupported();
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+
+  @WrapForJNI
+  public synchronized long input(
+      final ByteBuffer bytes, final BufferInfo info, final CryptoInfo cryptoInfo) {
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot send input to an ended codec");
+      return INVALID_SESSION;
+    }
+
+    final boolean eos = info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM;
+
+    if (eos) {
+      return sendInput(Sample.EOS);
+    }
+
+    try {
+      final Sample s = mRemote.dequeueInput(info.size);
+      fillInputBuffer(s.bufferId, bytes, info.offset, info.size);
+      mSession = s.session;
+      return sendInput(s.set(info, cryptoInfo));
+    } catch (final RemoteException | NullPointerException e) {
+      Log.e(LOGTAG, "fail to dequeue input buffer", e);
+    } catch (final IOException e) {
+      Log.e(LOGTAG, "fail to copy input data.", e);
+      // Balance dequeue/queue.
+      sendInput(null);
+    }
+    return INVALID_SESSION;
+  }
+
+  private void fillInputBuffer(
+      final int bufferId, final ByteBuffer bytes, final int offset, final int size)
+      throws RemoteException, IOException {
+    if (bytes == null || size == 0) {
+      Log.w(LOGTAG, "empty input");
+      return;
+    }
+
+    SampleBuffer buffer = mInputBuffers.get(bufferId);
+    if (buffer == null) {
+      buffer = mRemote.getInputBuffer(bufferId);
+      if (buffer != null) {
+        mInputBuffers.put(bufferId, buffer);
+      }
+    }
+
+    if (buffer.capacity() < size) {
+      final IOException e =
+          new IOException("data larger than capacity: " + size + " > " + buffer.capacity());
+      Log.e(LOGTAG, "cannot fill input.", e);
+      throw e;
+    }
+
+    buffer.readFromByteBuffer(bytes, offset, size);
+  }
+
+  private long sendInput(final Sample sample) {
+    try {
+      mRemote.queueInput(sample);
+      if (sample != null) {
+        sample.dispose();
+        mFlushed = false;
+      }
+    } catch (final Exception e) {
+      Log.e(LOGTAG, "fail to queue input:" + sample, e);
+      return INVALID_SESSION;
+    }
+    return mSession;
+  }
+
+  @WrapForJNI
+  public synchronized boolean flush() {
+    if (mFlushed) {
+      return true;
+    }
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot flush an ended codec");
+      return false;
+    }
+    try {
+      if (DEBUG) {
+        Log.d(LOGTAG, "flush " + this);
+      }
+      resetBuffers();
+      mRemote.flush();
+      mFlushed = true;
+    } catch (final DeadObjectException e) {
+      return false;
+    } catch (final RemoteException e) {
+      e.printStackTrace();
+      return false;
+    }
+    return true;
+  }
+
+  private void resetBuffers() {
+    for (int i = 0; i < mInputBuffers.size(); ++i) {
+      mInputBuffers.valueAt(i).dispose();
+    }
+    mInputBuffers.clear();
+    for (int i = 0; i < mOutputBuffers.size(); ++i) {
+      mOutputBuffers.valueAt(i).dispose();
+    }
+    mOutputBuffers.clear();
+  }
+
+  @WrapForJNI
+  public boolean release() {
+    mCallbacks.setCodecProxyReleased();
+    synchronized (this) {
+      if (mRemote == null) {
+        Log.w(LOGTAG, "codec already ended");
+        return true;
+      }
+      if (DEBUG) {
+        Log.d(LOGTAG, "release " + this);
+      }
+
+      if (!mSurfaceOutputs.isEmpty()) {
+        // Flushing output buffers to surface may cause some frames to be skipped and
+        // should not happen unless caller release codec before processing all buffers.
+        Log.w(LOGTAG, "release codec when " + mSurfaceOutputs.size() + " output buffers unhandled");
+        try {
+          for (final Sample s : mSurfaceOutputs) {
+            mRemote.releaseOutput(s, true);
+          }
+        } catch (final RemoteException e) {
+          e.printStackTrace();
+        }
+        mSurfaceOutputs.clear();
+      }
+
+      resetBuffers();
+
+      try {
+        RemoteManager.getInstance().releaseCodec(this);
+      } catch (final DeadObjectException e) {
+        return false;
+      } catch (final RemoteException e) {
+        e.printStackTrace();
+        return false;
+      }
+      return true;
+    }
+  }
+
+  @WrapForJNI
+  public synchronized boolean setBitrate(final int bps) {
+    if (!mIsEncoder) {
+      Log.w(LOGTAG, "this api is encoder-only");
+      return false;
+    }
+
+    if (android.os.Build.VERSION.SDK_INT < 19) {
+      Log.w(LOGTAG, "this api was added in API level 19");
+      return false;
+    }
+
+    if (mRemote == null) {
+      Log.w(LOGTAG, "codec already ended");
+      return true;
+    }
+
+    try {
+      mRemote.setBitrate(bps);
+    } catch (final RemoteException e) {
+      Log.e(LOGTAG, "remote fail to set rates:" + bps);
+      e.printStackTrace();
+    }
+    return true;
+  }
+
+  @WrapForJNI
+  public synchronized boolean releaseOutput(final Sample sample, final boolean render) {
+    if (mOutputSurface != null) {
+      if (!mSurfaceOutputs.remove(sample)) {
+        if (mRemote != null) Log.w(LOGTAG, "already released: " + sample);
+        return true;
+      }
+
+      if (DEBUG && !render) {
+        Log.d(LOGTAG, "drop output:" + sample.info.presentationTimeUs);
+      }
+    }
+
+    if (mRemote == null) {
+      Log.w(LOGTAG, "codec already ended");
+      sample.dispose();
+      return true;
+    }
+
+    try {
+      mRemote.releaseOutput(sample, render);
+    } catch (final RemoteException e) {
+      Log.e(LOGTAG, "remote fail to release output:" + sample.info.presentationTimeUs);
+      e.printStackTrace();
+    }
+    sample.dispose();
+
+    return true;
+  }
+
+  /* package */ void reportError(final boolean fatal) {
+    mCallbacks.reportError(fatal);
+  }
+
+  private synchronized SampleBuffer getOutputBuffer(final int id) {
+    if (mRemote == null) {
+      Log.e(LOGTAG, "cannot get buffer#" + id + " from an ended codec");
+      return null;
+    }
+
+    if (mOutputSurface != null || id == Sample.NO_BUFFER) {
+      return null;
+    }
+
+    SampleBuffer buffer = mOutputBuffers.get(id);
+    if (buffer != null) {
+      return buffer;
+    }
+
+    try {
+      buffer = mRemote.getOutputBuffer(id);
+    } catch (final Exception e) {
+      Log.e(LOGTAG, "cannot get buffer#" + id, e);
+      return null;
+    }
+    if (buffer != null) {
+      mOutputBuffers.put(id, buffer);
+    }
+
+    return buffer;
+  }
+
+  @WrapForJNI
+  public static boolean supportsCBCS() {
+    // Android N/API-24 supports CBCS but there seems to be a bug.
+    // See https://github.com/google/ExoPlayer/issues/4022
+    return Build.VERSION.SDK_INT >= Build.VERSION_CODES.N_MR1;
+  }
+
+  @RequiresApi(api = Build.VERSION_CODES.N_MR1)
+  @WrapForJNI
+  public static boolean setCryptoPatternIfNeeded(
+      final CryptoInfo info, final int blocksToEncrypt, final int blocksToSkip) {
+    if (supportsCBCS() && (blocksToEncrypt > 0 || blocksToSkip > 0)) {
+      info.setPattern(new CryptoInfo.Pattern(blocksToEncrypt, blocksToSkip));
+      return true;
+    }
+    return false;
+  }
+}