diff options
Diffstat (limited to 'mobile/android/geckoview/src/main/java/org/mozilla/gecko/media')
31 files changed, 7470 insertions, 0 deletions
diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodec.java new file mode 100644 index 0000000000..b29d488c6c --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodec.java @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.Handler; +import android.view.Surface; +import java.nio.ByteBuffer; + +// A wrapper interface that mimics the new {@link android.media.MediaCodec} +// asynchronous mode API in Lollipop. +public interface AsyncCodec { + interface Callbacks { + void onInputBufferAvailable(AsyncCodec codec, int index); + + void onOutputBufferAvailable(AsyncCodec codec, int index, BufferInfo info); + + void onError(AsyncCodec codec, int error); + + void onOutputFormatChanged(AsyncCodec codec, MediaFormat format); + } + + void setCallbacks(Callbacks callbacks, Handler handler); + + void configure(MediaFormat format, Surface surface, MediaCrypto crypto, int flags); + + boolean isAdaptivePlaybackSupported(String mimeType); + + boolean isTunneledPlaybackSupported(final String mimeType); + + void start(); + + void stop(); + + void flush(); + + // Must be called after flush(). + void resumeReceivingInputs(); + + void release(); + + ByteBuffer getInputBuffer(int index); + + MediaFormat getInputFormat(); + + ByteBuffer getOutputBuffer(int index); + + void queueInputBuffer(int index, int offset, int size, long presentationTimeUs, int flags); + + void setBitrate(int bps); + + void queueSecureInputBuffer( + int index, int offset, CryptoInfo info, long presentationTimeUs, int flags); + + void releaseOutputBuffer(int index, boolean render); +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodecFactory.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodecFactory.java new file mode 100644 index 0000000000..3295919b91 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodecFactory.java @@ -0,0 +1,19 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.os.Build; +import java.io.IOException; + +public final class AsyncCodecFactory { + public static AsyncCodec create(final String name) throws IOException { + // A bug that getInputBuffer() could fail after flush() then start() wasn't fixed until MR1. + // See: + // https://android.googlesource.com/platform/frameworks/av/+/d9e0603a1be07dbb347c55050c7d4629ea7492e8 + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1 + ? new LollipopAsyncCodec(name) + : new JellyBeanAsyncCodec(name); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/BaseHlsPlayer.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/BaseHlsPlayer.java new file mode 100644 index 0000000000..467d67681c --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/BaseHlsPlayer.java @@ -0,0 +1,104 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import java.util.concurrent.ConcurrentLinkedQueue; + +public interface BaseHlsPlayer { + + enum TrackType { + UNDEFINED, + AUDIO, + VIDEO, + TEXT, + } + + enum ResourceError { + BASE(-100), + UNKNOWN(-101), + PLAYER(-102), + UNSUPPORTED(-103); + + private int mNumVal; + + ResourceError(final int numVal) { + mNumVal = numVal; + } + + public int code() { + return mNumVal; + } + } + + enum DemuxerError { + BASE(-200), + UNKNOWN(-201), + PLAYER(-202), + UNSUPPORTED(-203); + + private int mNumVal; + + DemuxerError(final int numVal) { + mNumVal = numVal; + } + + public int code() { + return mNumVal; + } + } + + interface DemuxerCallbacks { + void onInitialized(boolean hasAudio, boolean hasVideo); + + void onError(int errorCode); + } + + interface ResourceCallbacks { + void onLoad(String mediaUrl); + + void onDataArrived(); + + void onError(int errorCode); + } + + // Used to identify player instance. + int getId(); + + // ======================================================================= + // API for GeckoHLSResourceWrapper + // ======================================================================= + void init(String url, ResourceCallbacks callback); + + boolean isLiveStream(); + + // ======================================================================= + // API for GeckoHLSDemuxerWrapper + // ======================================================================= + void addDemuxerWrapperCallbackListener(DemuxerCallbacks callback); + + ConcurrentLinkedQueue<GeckoHLSSample> getSamples(TrackType trackType, int number); + + long getBufferedPosition(); + + int getNumberOfTracks(TrackType trackType); + + GeckoVideoInfo getVideoInfo(int index); + + GeckoAudioInfo getAudioInfo(int index); + + boolean seek(long positionUs); + + long getNextKeyFrameTime(); + + void suspend(); + + void resume(); + + void play(); + + void pause(); + + void release(); +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java new file mode 100644 index 0000000000..eb07f6146c --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java @@ -0,0 +1,713 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.VideoCapabilities; +import android.media.MediaCodecList; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; +import android.view.Surface; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import org.mozilla.gecko.gfx.GeckoSurface; + +/* package */ final class Codec extends ICodec.Stub implements IBinder.DeathRecipient { + private static final String LOGTAG = "GeckoRemoteCodec"; + private static final boolean DEBUG = false; + public static final String SW_CODEC_PREFIX = "OMX.google."; + + public enum Error { + DECODE, + FATAL + } + + private final class Callbacks implements AsyncCodec.Callbacks { + @Override + public void onInputBufferAvailable(final AsyncCodec codec, final int index) { + mInputProcessor.onBuffer(index); + } + + @Override + public void onOutputBufferAvailable( + final AsyncCodec codec, final int index, final MediaCodec.BufferInfo info) { + mOutputProcessor.onBuffer(index, info); + } + + @Override + public void onError(final AsyncCodec codec, final int error) { + reportError(Error.FATAL, new Exception("codec error:" + error)); + } + + @Override + public void onOutputFormatChanged(final AsyncCodec codec, final MediaFormat format) { + mOutputProcessor.onFormatChanged(format); + } + } + + private static final class Input { + public final Sample sample; + public boolean reported; + + public Input(final Sample sample) { + this.sample = sample; + } + } + + private final class InputProcessor { + private boolean mHasInputCapacitySet; + private Queue<Integer> mAvailableInputBuffers = new LinkedList<>(); + private Queue<Sample> mDequeuedSamples = new LinkedList<>(); + private Queue<Input> mInputSamples = new LinkedList<>(); + private boolean mStopped; + + private synchronized Sample onAllocate(final int size) { + final Sample sample = mSamplePool.obtainInput(size); + sample.session = mSession; + mDequeuedSamples.add(sample); + return sample; + } + + private synchronized void onSample(final Sample sample) { + if (sample == null) { + // Ignore empty input. + mSamplePool.recycleInput(mDequeuedSamples.remove()); + Log.w(LOGTAG, "WARN: empty input sample"); + return; + } + + if (sample.isEOS()) { + queueSample(sample); + return; + } + + if (sample.session >= mSession) { + final Sample dequeued = mDequeuedSamples.remove(); + dequeued.setBufferInfo(sample.info); + dequeued.setCryptoInfo(sample.cryptoInfo); + queueSample(dequeued); + } + + sample.dispose(); + } + + private void queueSample(final Sample sample) { + if (!mInputSamples.offer(new Input(sample))) { + reportError(Error.FATAL, new Exception("FAIL: input sample queue is full")); + return; + } + + try { + feedSampleToBuffer(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + private synchronized void onBuffer(final int index) { + if (mStopped || !isValidBuffer(index)) { + return; + } + + if (!mHasInputCapacitySet) { + final int capacity = mCodec.getInputBuffer(index).capacity(); + if (capacity > 0) { + mSamplePool.setInputBufferSize(capacity); + mHasInputCapacitySet = true; + } + } + + if (mAvailableInputBuffers.offer(index)) { + feedSampleToBuffer(); + } else { + reportError(Error.FATAL, new Exception("FAIL: input buffer queue is full")); + } + } + + private boolean isValidBuffer(final int index) { + try { + return mCodec.getInputBuffer(index) != null; + } catch (final IllegalStateException e) { + if (DEBUG) { + Log.d(LOGTAG, "invalid input buffer#" + index, e); + } + return false; + } + } + + private void feedSampleToBuffer() { + while (!mAvailableInputBuffers.isEmpty() && !mInputSamples.isEmpty()) { + final int index = mAvailableInputBuffers.poll(); + if (!isValidBuffer(index)) { + continue; + } + int len = 0; + final Sample sample = mInputSamples.poll().sample; + final long pts = sample.info.presentationTimeUs; + final int flags = sample.info.flags; + final MediaCodec.CryptoInfo cryptoInfo = sample.cryptoInfo; + if (!sample.isEOS() && sample.bufferId != Sample.NO_BUFFER) { + len = sample.info.size; + final ByteBuffer buf = mCodec.getInputBuffer(index); + try { + mSamplePool + .getInputBuffer(sample.bufferId) + .writeToByteBuffer(buf, sample.info.offset, len); + } catch (final IOException e) { + e.printStackTrace(); + len = 0; + } + mSamplePool.recycleInput(sample); + } + + try { + if (cryptoInfo != null && len > 0) { + mCodec.queueSecureInputBuffer(index, 0, cryptoInfo, pts, flags); + } else { + mCodec.queueInputBuffer(index, 0, len, pts, flags); + } + mCallbacks.onInputQueued(pts); + } catch (final RemoteException e) { + e.printStackTrace(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + return; + } + } + reportPendingInputs(); + } + + private void reportPendingInputs() { + try { + for (final Input i : mInputSamples) { + if (!i.reported) { + i.reported = true; + mCallbacks.onInputPending(i.sample.info.presentationTimeUs); + } + } + } catch (final RemoteException e) { + e.printStackTrace(); + } + } + + private synchronized void reset() { + for (final Input i : mInputSamples) { + if (!i.sample.isEOS()) { + mSamplePool.recycleInput(i.sample); + } + } + mInputSamples.clear(); + + for (final Sample s : mDequeuedSamples) { + mSamplePool.recycleInput(s); + } + mDequeuedSamples.clear(); + + mAvailableInputBuffers.clear(); + } + + private synchronized void start() { + if (!mStopped) { + return; + } + mStopped = false; + } + + private synchronized void stop() { + if (mStopped) { + return; + } + mStopped = true; + reset(); + } + } + + private static final class Output { + public final Sample sample; + public final int index; + + public Output(final Sample sample, final int index) { + this.sample = sample; + this.index = index; + } + } + + private class OutputProcessor { + private final boolean mRenderToSurface; + private boolean mHasOutputCapacitySet; + private Queue<Output> mSentOutputs = new LinkedList<>(); + private boolean mStopped; + + private OutputProcessor(final boolean renderToSurface) { + mRenderToSurface = renderToSurface; + } + + private synchronized void onBuffer(final int index, final MediaCodec.BufferInfo info) { + if (mStopped || !isValidBuffer(index)) { + return; + } + + try { + final Sample output = obtainOutputSample(index, info); + mSentOutputs.add(new Output(output, index)); + output.session = mSession; + mCallbacks.onOutput(output); + } catch (final Exception e) { + e.printStackTrace(); + mCodec.releaseOutputBuffer(index, false); + } + + final boolean eos = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + if (DEBUG && eos) { + Log.d(LOGTAG, "output EOS"); + } + } + + private boolean isValidBuffer(final int index) { + try { + return (mCodec.getOutputBuffer(index) != null) || mRenderToSurface; + } catch (final IllegalStateException e) { + if (DEBUG) { + Log.e(LOGTAG, "invalid buffer#" + index, e); + } + return false; + } + } + + private Sample obtainOutputSample(final int index, final MediaCodec.BufferInfo info) { + final Sample sample = mSamplePool.obtainOutput(info); + + if (mRenderToSurface) { + return sample; + } + + final ByteBuffer output = mCodec.getOutputBuffer(index); + if (!mHasOutputCapacitySet) { + final int capacity = output.capacity(); + if (capacity > 0) { + mSamplePool.setOutputBufferSize(capacity); + mHasOutputCapacitySet = true; + } + } + + if (info.size > 0) { + try { + mSamplePool + .getOutputBuffer(sample.bufferId) + .readFromByteBuffer(output, info.offset, info.size); + } catch (final IOException e) { + Log.e(LOGTAG, "Fail to read output buffer:" + e.getMessage()); + } + } + + return sample; + } + + private synchronized void onRelease(final Sample sample, final boolean render) { + final Output output = mSentOutputs.poll(); + if (output != null) { + mCodec.releaseOutputBuffer(output.index, render); + mSamplePool.recycleOutput(output.sample); + } else if (DEBUG) { + Log.d(LOGTAG, sample + " already released"); + } + + sample.dispose(); + } + + private synchronized void onFormatChanged(final MediaFormat format) { + if (mStopped) { + return; + } + try { + mCallbacks.onOutputFormatChanged(new FormatParam(format)); + } catch (final RemoteException re) { + // Dead recipient. + re.printStackTrace(); + } + } + + private synchronized void reset() { + for (final Output o : mSentOutputs) { + mCodec.releaseOutputBuffer(o.index, false); + mSamplePool.recycleOutput(o.sample); + } + mSentOutputs.clear(); + } + + private synchronized void start() { + if (!mStopped) { + return; + } + mStopped = false; + } + + private synchronized void stop() { + if (mStopped) { + return; + } + mStopped = true; + reset(); + } + } + + private volatile ICodecCallbacks mCallbacks; + private GeckoSurface mSurface; + private AsyncCodec mCodec; + private InputProcessor mInputProcessor; + private OutputProcessor mOutputProcessor; + private long mSession; + private SamplePool mSamplePool; + // Values will be updated after configure called. + private volatile boolean mIsAdaptivePlaybackSupported = false; + private volatile boolean mIsHardwareAccelerated = false; + private boolean mIsTunneledPlaybackSupported = false; + + public synchronized void setCallbacks(final ICodecCallbacks callbacks) throws RemoteException { + mCallbacks = callbacks; + callbacks.asBinder().linkToDeath(this, 0); + } + + // IBinder.DeathRecipient + @Override + public synchronized void binderDied() { + Log.e(LOGTAG, "Callbacks is dead"); + try { + release(); + } catch (final RemoteException e) { + // Nowhere to report the error. + } + } + + @Override + public synchronized boolean configure( + final FormatParam format, final GeckoSurface surface, final int flags, final String drmStubId) + throws RemoteException { + if (mCallbacks == null) { + Log.e(LOGTAG, "FAIL: callbacks must be set before calling configure()"); + return false; + } + + if (mCodec != null) { + if (DEBUG) { + Log.d(LOGTAG, "release existing codec: " + mCodec); + } + mCodec.release(); + } + + if (DEBUG) { + Log.d(LOGTAG, "configure " + this); + } + + final MediaFormat fmt = format.asFormat(); + final String mime = fmt.getString(MediaFormat.KEY_MIME); + if (mime == null || mime.isEmpty()) { + Log.e(LOGTAG, "invalid MIME type: " + mime); + return false; + } + + final List<String> found = + findMatchingCodecNames(fmt, flags == MediaCodec.CONFIGURE_FLAG_ENCODE); + for (final String name : found) { + final AsyncCodec codec = + configureCodec( + name, fmt, surface != null ? surface.getSurface() : null, flags, drmStubId); + if (codec == null) { + Log.w(LOGTAG, "unable to configure " + name + ". Try next."); + continue; + } + mIsHardwareAccelerated = !name.startsWith(SW_CODEC_PREFIX); + mCodec = codec; + // Bug 1789846: Check if the Codec provides stride or height values to use. + if (flags == MediaCodec.CONFIGURE_FLAG_ENCODE && fmt.containsKey(MediaFormat.KEY_WIDTH)) { + final MediaFormat inputFormat = mCodec.getInputFormat(); + if (inputFormat != null) { + if (inputFormat.containsKey(MediaFormat.KEY_STRIDE)) { + fmt.setInteger(MediaFormat.KEY_STRIDE, inputFormat.getInteger(MediaFormat.KEY_STRIDE)); + } + if (inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + fmt.setInteger( + MediaFormat.KEY_SLICE_HEIGHT, inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT)); + } + } + } + mInputProcessor = new InputProcessor(); + final boolean renderToSurface = surface != null; + mOutputProcessor = new OutputProcessor(renderToSurface); + mSamplePool = new SamplePool(name, renderToSurface); + if (renderToSurface) { + mIsTunneledPlaybackSupported = mCodec.isTunneledPlaybackSupported(mime); + mSurface = surface; // Take ownership of surface. + } + if (DEBUG) { + Log.d(LOGTAG, codec.toString() + " created. Render to surface?" + renderToSurface); + } + return true; + } + + return false; + } + + private List<String> findMatchingCodecNames(final MediaFormat format, final boolean isEncoder) { + final String mimeType = format.getString(MediaFormat.KEY_MIME); + // Missing width and height value in format means audio; + // Video format should never has 0 width or height. + final int width = + format.containsKey(MediaFormat.KEY_WIDTH) ? format.getInteger(MediaFormat.KEY_WIDTH) : 0; + final int height = + format.containsKey(MediaFormat.KEY_HEIGHT) ? format.getInteger(MediaFormat.KEY_HEIGHT) : 0; + + int numCodecs = 0; + final List<String> found = new ArrayList<>(); + try { + numCodecs = MediaCodecList.getCodecCount(); + } catch (final RuntimeException e) { + Log.e(LOGTAG, "Failed retrieving codec count finding matching codec names", e); + return found; + } + + for (int i = 0; i < numCodecs; i++) { + final MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); + if (info.isEncoder() == !isEncoder) { + continue; + } + + final String[] types = info.getSupportedTypes(); + for (final String t : types) { + if (!t.equalsIgnoreCase(mimeType)) { + continue; + } + final String name = info.getName(); + // API 21+ provide a method to query whether supplied size is supported. For + // older version, just avoid software video encoders. + if (isEncoder && width > 0 && height > 0) { + final VideoCapabilities c = info.getCapabilitiesForType(mimeType).getVideoCapabilities(); + if (c != null && !c.isSizeSupported(width, height)) { + if (DEBUG) { + Log.d(LOGTAG, name + ": " + width + "x" + height + " not supported"); + } + continue; + } + } + + found.add(name); + if (DEBUG) { + Log.d( + LOGTAG, + "found " + (isEncoder ? "encoder:" : "decoder:") + name + " for mime:" + mimeType); + } + } + } + return found; + } + + private AsyncCodec configureCodec( + final String name, + final MediaFormat format, + final Surface surface, + final int flags, + final String drmStubId) { + try { + final AsyncCodec codec = AsyncCodecFactory.create(name); + codec.setCallbacks(new Callbacks(), null); + + final MediaCrypto crypto = RemoteMediaDrmBridgeStub.getMediaCrypto(drmStubId); + if (DEBUG) { + Log.d( + LOGTAG, + "configure mediacodec with crypto(" + (crypto != null) + ") / Id :" + drmStubId); + } + + if (surface != null) { + setupAdaptivePlayback(codec, format); + } + + codec.configure(format, surface, crypto, flags); + return codec; + } catch (final Exception e) { + Log.e(LOGTAG, "codec creation error", e); + return null; + } + } + + private void setupAdaptivePlayback(final AsyncCodec codec, final MediaFormat format) { + // Video decoder should config with adaptive playback capability. + mIsAdaptivePlaybackSupported = + codec.isAdaptivePlaybackSupported(format.getString(MediaFormat.KEY_MIME)); + if (mIsAdaptivePlaybackSupported) { + if (DEBUG) { + Log.d(LOGTAG, "codec supports adaptive playback = " + mIsAdaptivePlaybackSupported); + } + // TODO: may need to find a way to not use hard code to decide the max w/h. + format.setInteger(MediaFormat.KEY_MAX_WIDTH, 1920); + format.setInteger(MediaFormat.KEY_MAX_HEIGHT, 1080); + } + } + + @Override + public synchronized boolean isAdaptivePlaybackSupported() { + return mIsAdaptivePlaybackSupported; + } + + @Override + public synchronized boolean isHardwareAccelerated() { + return mIsHardwareAccelerated; + } + + @Override + public synchronized boolean isTunneledPlaybackSupported() { + return mIsTunneledPlaybackSupported; + } + + @Override + public synchronized void start() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "start " + this); + } + mInputProcessor.start(); + mOutputProcessor.start(); + try { + mCodec.start(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + private void reportError(final Error error, final Exception e) { + if (e != null) { + e.printStackTrace(); + } + try { + mCallbacks.onError(error == Error.FATAL); + } catch (final NullPointerException ne) { + // mCallbacks has been disposed by release(). + } catch (final RemoteException re) { + re.printStackTrace(); + } + } + + @Override + public synchronized void stop() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "stop " + this); + } + try { + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.stop(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void flush() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "flush " + this); + } + try { + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.flush(); + if (DEBUG) { + Log.d(LOGTAG, "flushed " + this); + } + mInputProcessor.start(); + mOutputProcessor.start(); + mCodec.resumeReceivingInputs(); + mSession++; + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized Sample dequeueInput(final int size) throws RemoteException { + try { + return mInputProcessor.onAllocate(size); + } catch (final Exception e) { + // Translate allocation error to remote exception. + throw new RemoteException(e.getMessage()); + } + } + + @Override + public synchronized SampleBuffer getInputBuffer(final int id) { + if (mSamplePool == null) { + return null; + } + return mSamplePool.getInputBuffer(id); + } + + @Override + public synchronized SampleBuffer getOutputBuffer(final int id) { + if (mSamplePool == null) { + return null; + } + return mSamplePool.getOutputBuffer(id); + } + + @Override + public synchronized void queueInput(final Sample sample) throws RemoteException { + try { + mInputProcessor.onSample(sample); + } catch (final Exception e) { + throw new RemoteException(e.getMessage()); + } + } + + @Override + public synchronized void setBitrate(final int bps) { + try { + mCodec.setBitrate(bps); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void releaseOutput(final Sample sample, final boolean render) { + try { + mOutputProcessor.onRelease(sample, render); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + } + + @Override + public synchronized void release() throws RemoteException { + if (DEBUG) { + Log.d(LOGTAG, "release " + this); + } + try { + // In case Codec.stop() is not called yet. + mInputProcessor.stop(); + mOutputProcessor.stop(); + + mCodec.release(); + } catch (final Exception e) { + reportError(Error.FATAL, e); + } + mCodec = null; + mSamplePool.reset(); + mSamplePool = null; + mCallbacks.asBinder().unlinkToDeath(this, 0); + mCallbacks = null; + if (mSurface != null) { + mSurface.release(); + mSurface = null; + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java new file mode 100644 index 0000000000..34bba3e593 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java @@ -0,0 +1,503 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import android.media.MediaFormat; +import android.os.Build; +import android.os.DeadObjectException; +import android.os.RemoteException; +import android.util.Log; +import android.util.SparseArray; +import androidx.annotation.RequiresApi; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.mozilla.gecko.annotation.WrapForJNI; +import org.mozilla.gecko.gfx.GeckoSurface; +import org.mozilla.gecko.mozglue.JNIObject; + +// Proxy class of ICodec binder. +public final class CodecProxy { + private static final String LOGTAG = "GeckoRemoteCodecProxy"; + private static final boolean DEBUG = false; + @WrapForJNI private static final long INVALID_SESSION = -1; + + private ICodec mRemote; + private long mSession; + private boolean mIsEncoder; + private FormatParam mFormat; + private GeckoSurface mOutputSurface; + private CallbacksForwarder mCallbacks; + private String mRemoteDrmStubId; + private Queue<Sample> mSurfaceOutputs = new ConcurrentLinkedQueue<>(); + private boolean mFlushed = true; + + private SparseArray<SampleBuffer> mInputBuffers = new SparseArray<>(); + private SparseArray<SampleBuffer> mOutputBuffers = new SparseArray<>(); + + public interface Callbacks { + void onInputStatus(long timestamp, boolean processed); + + void onOutputFormatChanged(MediaFormat format); + + void onOutput(Sample output, SampleBuffer buffer); + + void onError(boolean fatal); + } + + @WrapForJNI + public static class NativeCallbacks extends JNIObject implements Callbacks { + public native void onInputStatus(long timestamp, boolean processed); + + public native void onOutputFormatChanged(MediaFormat format); + + public native void onOutput(Sample output, SampleBuffer buffer); + + public native void onError(boolean fatal); + + @Override // JNIObject + protected void disposeNative() { + throw new UnsupportedOperationException(); + } + } + + private class CallbacksForwarder extends ICodecCallbacks.Stub { + private final Callbacks mCallbacks; + private boolean mCodecProxyReleased; + + CallbacksForwarder(final Callbacks callbacks) { + mCallbacks = callbacks; + } + + @Override + public synchronized void onInputQueued(final long timestamp) throws RemoteException { + if (!mCodecProxyReleased) { + mCallbacks.onInputStatus(timestamp, true /* processed */); + } + } + + @Override + public synchronized void onInputPending(final long timestamp) throws RemoteException { + if (!mCodecProxyReleased) { + mCallbacks.onInputStatus(timestamp, false /* processed */); + } + } + + @Override + public synchronized void onOutputFormatChanged(final FormatParam format) + throws RemoteException { + if (!mCodecProxyReleased) { + mCallbacks.onOutputFormatChanged(format.asFormat()); + } + } + + @Override + public synchronized void onOutput(final Sample sample) throws RemoteException { + if (mCodecProxyReleased) { + sample.dispose(); + return; + } + + final SampleBuffer buffer = CodecProxy.this.getOutputBuffer(sample.bufferId); + if (mOutputSurface != null) { + // Don't render to surface just yet. Callback will make that happen when it's time. + mSurfaceOutputs.offer(sample); + } else if (buffer == null) { + // Buffer with given ID has been flushed. + sample.dispose(); + return; + } + mCallbacks.onOutput(sample, buffer); + } + + @Override + public void onError(final boolean fatal) throws RemoteException { + reportError(fatal); + } + + private synchronized void reportError(final boolean fatal) { + if (!mCodecProxyReleased) { + mCallbacks.onError(fatal); + } + } + + private synchronized void setCodecProxyReleased() { + mCodecProxyReleased = true; + } + } + + @WrapForJNI + public int GetInputFormatStride() { + if (mFormat.asFormat().containsKey(MediaFormat.KEY_STRIDE)) { + return mFormat.asFormat().getInteger(MediaFormat.KEY_STRIDE); + } + return 0; + } + + @WrapForJNI + public int GetInputFormatYPlaneHeight() { + if (mFormat.asFormat().containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + return mFormat.asFormat().getInteger(MediaFormat.KEY_SLICE_HEIGHT); + } + return 0; + } + + @WrapForJNI + public static CodecProxy create( + final boolean isEncoder, + final MediaFormat format, + final GeckoSurface surface, + final Callbacks callbacks, + final String drmStubId) { + return RemoteManager.getInstance() + .createCodec(isEncoder, format, surface, callbacks, drmStubId); + } + + public static CodecProxy createCodecProxy( + final boolean isEncoder, + final MediaFormat format, + final GeckoSurface surface, + final Callbacks callbacks, + final String drmStubId) { + return new CodecProxy(isEncoder, format, surface, callbacks, drmStubId); + } + + private CodecProxy( + final boolean isEncoder, + final MediaFormat format, + final GeckoSurface surface, + final Callbacks callbacks, + final String drmStubId) { + mIsEncoder = isEncoder; + mFormat = new FormatParam(format); + mOutputSurface = surface; + mRemoteDrmStubId = drmStubId; + mCallbacks = new CallbacksForwarder(callbacks); + } + + boolean init(final ICodec remote) { + try { + remote.setCallbacks(mCallbacks); + if (!remote.configure( + mFormat, + mOutputSurface, + mIsEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0, + mRemoteDrmStubId)) { + return false; + } + remote.start(); + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + + mRemote = remote; + return true; + } + + boolean deinit() { + try { + mRemote.stop(); + mRemote.release(); + mRemote = null; + return true; + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + } + + @WrapForJNI + public synchronized boolean isAdaptivePlaybackSupported() { + if (mRemote == null) { + Log.e(LOGTAG, "cannot check isAdaptivePlaybackSupported with an ended codec"); + return false; + } + try { + return mRemote.isAdaptivePlaybackSupported(); + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + } + + @WrapForJNI + public synchronized boolean isHardwareAccelerated() { + if (mRemote == null) { + Log.e(LOGTAG, "cannot check isHardwareAccelerated with an ended codec"); + return false; + } + try { + return mRemote.isHardwareAccelerated(); + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + } + + @WrapForJNI + public synchronized boolean isTunneledPlaybackSupported() { + if (mRemote == null) { + Log.e(LOGTAG, "cannot check isTunneledPlaybackSupported with an ended codec"); + return false; + } + try { + return mRemote.isTunneledPlaybackSupported(); + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + } + + @WrapForJNI + public synchronized long input( + final ByteBuffer bytes, final BufferInfo info, final CryptoInfo cryptoInfo) { + if (mRemote == null) { + Log.e(LOGTAG, "cannot send input to an ended codec"); + return INVALID_SESSION; + } + + final boolean eos = info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM; + + if (eos) { + return sendInput(Sample.EOS); + } + + try { + final Sample s = mRemote.dequeueInput(info.size); + fillInputBuffer(s.bufferId, bytes, info.offset, info.size); + mSession = s.session; + return sendInput(s.set(info, cryptoInfo)); + } catch (final RemoteException | NullPointerException e) { + Log.e(LOGTAG, "fail to dequeue input buffer", e); + } catch (final IOException e) { + Log.e(LOGTAG, "fail to copy input data.", e); + // Balance dequeue/queue. + sendInput(null); + } + return INVALID_SESSION; + } + + private void fillInputBuffer( + final int bufferId, final ByteBuffer bytes, final int offset, final int size) + throws RemoteException, IOException { + if (bytes == null || size == 0) { + Log.w(LOGTAG, "empty input"); + return; + } + + SampleBuffer buffer = mInputBuffers.get(bufferId); + if (buffer == null) { + buffer = mRemote.getInputBuffer(bufferId); + if (buffer != null) { + mInputBuffers.put(bufferId, buffer); + } + } + + if (buffer.capacity() < size) { + final IOException e = + new IOException("data larger than capacity: " + size + " > " + buffer.capacity()); + Log.e(LOGTAG, "cannot fill input.", e); + throw e; + } + + buffer.readFromByteBuffer(bytes, offset, size); + } + + private long sendInput(final Sample sample) { + try { + mRemote.queueInput(sample); + if (sample != null) { + sample.dispose(); + mFlushed = false; + } + } catch (final Exception e) { + Log.e(LOGTAG, "fail to queue input:" + sample, e); + return INVALID_SESSION; + } + return mSession; + } + + @WrapForJNI + public synchronized boolean flush() { + if (mFlushed) { + return true; + } + if (mRemote == null) { + Log.e(LOGTAG, "cannot flush an ended codec"); + return false; + } + try { + if (DEBUG) { + Log.d(LOGTAG, "flush " + this); + } + resetBuffers(); + mRemote.flush(); + mFlushed = true; + } catch (final DeadObjectException e) { + return false; + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + return true; + } + + private void resetBuffers() { + for (int i = 0; i < mInputBuffers.size(); ++i) { + mInputBuffers.valueAt(i).dispose(); + } + mInputBuffers.clear(); + for (int i = 0; i < mOutputBuffers.size(); ++i) { + mOutputBuffers.valueAt(i).dispose(); + } + mOutputBuffers.clear(); + } + + @WrapForJNI + public boolean release() { + mCallbacks.setCodecProxyReleased(); + synchronized (this) { + if (mRemote == null) { + Log.w(LOGTAG, "codec already ended"); + return true; + } + if (DEBUG) { + Log.d(LOGTAG, "release " + this); + } + + if (!mSurfaceOutputs.isEmpty()) { + // Flushing output buffers to surface may cause some frames to be skipped and + // should not happen unless caller release codec before processing all buffers. + Log.w(LOGTAG, "release codec when " + mSurfaceOutputs.size() + " output buffers unhandled"); + try { + for (final Sample s : mSurfaceOutputs) { + mRemote.releaseOutput(s, true); + } + } catch (final RemoteException e) { + e.printStackTrace(); + } + mSurfaceOutputs.clear(); + } + + resetBuffers(); + + try { + RemoteManager.getInstance().releaseCodec(this); + } catch (final DeadObjectException e) { + return false; + } catch (final RemoteException e) { + e.printStackTrace(); + return false; + } + return true; + } + } + + @WrapForJNI + public synchronized boolean setBitrate(final int bps) { + if (!mIsEncoder) { + Log.w(LOGTAG, "this api is encoder-only"); + return false; + } + + if (mRemote == null) { + Log.w(LOGTAG, "codec already ended"); + return true; + } + + try { + mRemote.setBitrate(bps); + } catch (final RemoteException e) { + Log.e(LOGTAG, "remote fail to set rates:" + bps); + e.printStackTrace(); + } + return true; + } + + @WrapForJNI + public synchronized boolean releaseOutput(final Sample sample, final boolean render) { + if (mOutputSurface != null) { + if (!mSurfaceOutputs.remove(sample)) { + if (mRemote != null) Log.w(LOGTAG, "already released: " + sample); + return true; + } + + if (DEBUG && !render) { + Log.d(LOGTAG, "drop output:" + sample.info.presentationTimeUs); + } + } + + if (mRemote == null) { + Log.w(LOGTAG, "codec already ended"); + sample.dispose(); + return true; + } + + try { + mRemote.releaseOutput(sample, render); + } catch (final RemoteException e) { + Log.e(LOGTAG, "remote fail to release output:" + sample.info.presentationTimeUs); + e.printStackTrace(); + } + sample.dispose(); + + return true; + } + + /* package */ void reportError(final boolean fatal) { + mCallbacks.reportError(fatal); + } + + private synchronized SampleBuffer getOutputBuffer(final int id) { + if (mRemote == null) { + Log.e(LOGTAG, "cannot get buffer#" + id + " from an ended codec"); + return null; + } + + if (mOutputSurface != null || id == Sample.NO_BUFFER) { + return null; + } + + SampleBuffer buffer = mOutputBuffers.get(id); + if (buffer != null) { + return buffer; + } + + try { + buffer = mRemote.getOutputBuffer(id); + } catch (final Exception e) { + Log.e(LOGTAG, "cannot get buffer#" + id, e); + return null; + } + if (buffer != null) { + mOutputBuffers.put(id, buffer); + } + + return buffer; + } + + @WrapForJNI + public static boolean supportsCBCS() { + // Android N/API-24 supports CBCS but there seems to be a bug. + // See https://github.com/google/ExoPlayer/issues/4022 + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.N_MR1; + } + + @RequiresApi(api = Build.VERSION_CODES.N_MR1) + @WrapForJNI + public static boolean setCryptoPatternIfNeeded( + final CryptoInfo info, final int blocksToEncrypt, final int blocksToSkip) { + if (supportsCBCS() && (blocksToEncrypt > 0 || blocksToSkip > 0)) { + info.setPattern(new CryptoInfo.Pattern(blocksToEncrypt, blocksToSkip)); + return true; + } + return false; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/FormatParam.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/FormatParam.java new file mode 100644 index 0000000000..99287974f5 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/FormatParam.java @@ -0,0 +1,199 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaFormat; +import android.os.Build; +import android.os.Bundle; +import android.os.Parcel; +import android.os.Parcelable; +import java.nio.ByteBuffer; + +/** + * A wrapper to make {@link MediaFormat} parcelable. Supports following keys: + * + * <ul> + * <li>{@link MediaFormat#KEY_MIME} + * <li>{@link MediaFormat#KEY_WIDTH} + * <li>{@link MediaFormat#KEY_HEIGHT} + * <li>{@link MediaFormat#KEY_CHANNEL_COUNT} + * <li>{@link MediaFormat#KEY_SAMPLE_RATE} + * <li>{@link MediaFormat#KEY_BIT_RATE} + * <li>{@link MediaFormat#KEY_BITRATE_MODE} + * <li>{@link MediaFormat#KEY_COLOR_FORMAT} + * <li>{@link MediaFormat#KEY_FRAME_RATE} + * <li>{@link MediaFormat#KEY_I_FRAME_INTERVAL} + * <li>{@link MediaFormat#KEY_STRIDE} + * <li>{@link MediaFormat#KEY_SLICE_HEIGHT} + * <li>{@link MediaFormat#KEY_COLOR_RANGE + * <li>{@link MediaFormat#KEY_COLOR_STANDARD} + * <li>"csd-0" + * <li>"csd-1" + * </ul> + */ +public final class FormatParam implements Parcelable { + // Keys for codec specific config bits not exposed in {@link MediaFormat}. + private static final String KEY_CONFIG_0 = "csd-0"; + private static final String KEY_CONFIG_1 = "csd-1"; + + private MediaFormat mFormat; + + public MediaFormat asFormat() { + return mFormat; + } + + public FormatParam(final MediaFormat format) { + mFormat = format; + } + + protected FormatParam(final Parcel in) { + mFormat = new MediaFormat(); + readFromParcel(in); + } + + public static final Creator<FormatParam> CREATOR = + new Creator<FormatParam>() { + @Override + public FormatParam createFromParcel(final Parcel in) { + return new FormatParam(in); + } + + @Override + public FormatParam[] newArray(final int size) { + return new FormatParam[size]; + } + }; + + @Override + public int describeContents() { + return 0; + } + + public void readFromParcel(final Parcel in) { + final Bundle bundle = in.readBundle(); + fromBundle(bundle); + } + + private void fromBundle(final Bundle bundle) { + if (bundle.containsKey(MediaFormat.KEY_MIME)) { + mFormat.setString(MediaFormat.KEY_MIME, bundle.getString(MediaFormat.KEY_MIME)); + } + if (bundle.containsKey(MediaFormat.KEY_WIDTH)) { + mFormat.setInteger(MediaFormat.KEY_WIDTH, bundle.getInt(MediaFormat.KEY_WIDTH)); + } + if (bundle.containsKey(MediaFormat.KEY_HEIGHT)) { + mFormat.setInteger(MediaFormat.KEY_HEIGHT, bundle.getInt(MediaFormat.KEY_HEIGHT)); + } + if (bundle.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { + mFormat.setInteger( + MediaFormat.KEY_CHANNEL_COUNT, bundle.getInt(MediaFormat.KEY_CHANNEL_COUNT)); + } + if (bundle.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { + mFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, bundle.getInt(MediaFormat.KEY_SAMPLE_RATE)); + } + if (bundle.containsKey(KEY_CONFIG_0)) { + mFormat.setByteBuffer(KEY_CONFIG_0, ByteBuffer.wrap(bundle.getByteArray(KEY_CONFIG_0))); + } + if (bundle.containsKey(KEY_CONFIG_1)) { + mFormat.setByteBuffer(KEY_CONFIG_1, ByteBuffer.wrap(bundle.getByteArray((KEY_CONFIG_1)))); + } + if (bundle.containsKey(MediaFormat.KEY_BIT_RATE)) { + mFormat.setInteger(MediaFormat.KEY_BIT_RATE, bundle.getInt(MediaFormat.KEY_BIT_RATE)); + } + if (bundle.containsKey(MediaFormat.KEY_BITRATE_MODE)) { + mFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, bundle.getInt(MediaFormat.KEY_BITRATE_MODE)); + } + if (bundle.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { + mFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, bundle.getInt(MediaFormat.KEY_COLOR_FORMAT)); + } + if (bundle.containsKey(MediaFormat.KEY_FRAME_RATE)) { + mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, bundle.getInt(MediaFormat.KEY_FRAME_RATE)); + } + if (bundle.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { + mFormat.setInteger( + MediaFormat.KEY_I_FRAME_INTERVAL, bundle.getInt(MediaFormat.KEY_I_FRAME_INTERVAL)); + } + if (bundle.containsKey(MediaFormat.KEY_STRIDE)) { + mFormat.setInteger(MediaFormat.KEY_STRIDE, bundle.getInt(MediaFormat.KEY_STRIDE)); + } + if (bundle.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + mFormat.setInteger(MediaFormat.KEY_SLICE_HEIGHT, bundle.getInt(MediaFormat.KEY_SLICE_HEIGHT)); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + if (bundle.containsKey(MediaFormat.KEY_COLOR_RANGE)) { + mFormat.setInteger(MediaFormat.KEY_COLOR_RANGE, bundle.getInt(MediaFormat.KEY_COLOR_RANGE)); + } + if (bundle.containsKey(MediaFormat.KEY_COLOR_STANDARD)) { + mFormat.setInteger( + MediaFormat.KEY_COLOR_STANDARD, bundle.getInt(MediaFormat.KEY_COLOR_STANDARD)); + } + } + } + + @Override + public void writeToParcel(final Parcel dest, final int flags) { + dest.writeBundle(toBundle()); + } + + private Bundle toBundle() { + final Bundle bundle = new Bundle(); + if (mFormat.containsKey(MediaFormat.KEY_MIME)) { + bundle.putString(MediaFormat.KEY_MIME, mFormat.getString(MediaFormat.KEY_MIME)); + } + if (mFormat.containsKey(MediaFormat.KEY_WIDTH)) { + bundle.putInt(MediaFormat.KEY_WIDTH, mFormat.getInteger(MediaFormat.KEY_WIDTH)); + } + if (mFormat.containsKey(MediaFormat.KEY_HEIGHT)) { + bundle.putInt(MediaFormat.KEY_HEIGHT, mFormat.getInteger(MediaFormat.KEY_HEIGHT)); + } + if (mFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { + bundle.putInt( + MediaFormat.KEY_CHANNEL_COUNT, mFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + } + if (mFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { + bundle.putInt(MediaFormat.KEY_SAMPLE_RATE, mFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); + } + if (mFormat.containsKey(KEY_CONFIG_0)) { + final ByteBuffer bytes = mFormat.getByteBuffer(KEY_CONFIG_0); + bundle.putByteArray(KEY_CONFIG_0, Sample.byteArrayFromBuffer(bytes, 0, bytes.capacity())); + } + if (mFormat.containsKey(KEY_CONFIG_1)) { + final ByteBuffer bytes = mFormat.getByteBuffer(KEY_CONFIG_1); + bundle.putByteArray(KEY_CONFIG_1, Sample.byteArrayFromBuffer(bytes, 0, bytes.capacity())); + } + if (mFormat.containsKey(MediaFormat.KEY_BIT_RATE)) { + bundle.putInt(MediaFormat.KEY_BIT_RATE, mFormat.getInteger(MediaFormat.KEY_BIT_RATE)); + } + if (mFormat.containsKey(MediaFormat.KEY_BITRATE_MODE)) { + bundle.putInt(MediaFormat.KEY_BITRATE_MODE, mFormat.getInteger(MediaFormat.KEY_BITRATE_MODE)); + } + if (mFormat.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { + bundle.putInt(MediaFormat.KEY_COLOR_FORMAT, mFormat.getInteger(MediaFormat.KEY_COLOR_FORMAT)); + } + if (mFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) { + bundle.putInt(MediaFormat.KEY_FRAME_RATE, mFormat.getInteger(MediaFormat.KEY_FRAME_RATE)); + } + if (mFormat.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { + bundle.putInt( + MediaFormat.KEY_I_FRAME_INTERVAL, mFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)); + } + if (mFormat.containsKey(MediaFormat.KEY_STRIDE)) { + bundle.putInt(MediaFormat.KEY_STRIDE, mFormat.getInteger(MediaFormat.KEY_STRIDE)); + } + if (mFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + bundle.putInt(MediaFormat.KEY_SLICE_HEIGHT, mFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT)); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + if (mFormat.containsKey(MediaFormat.KEY_COLOR_RANGE)) { + bundle.putInt(MediaFormat.KEY_COLOR_RANGE, mFormat.getInteger(MediaFormat.KEY_COLOR_RANGE)); + } + if (mFormat.containsKey(MediaFormat.KEY_COLOR_STANDARD)) { + bundle.putInt( + MediaFormat.KEY_COLOR_STANDARD, mFormat.getInteger(MediaFormat.KEY_COLOR_STANDARD)); + } + } + return bundle; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoAudioInfo.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoAudioInfo.java new file mode 100644 index 0000000000..6418375a57 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoAudioInfo.java @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import org.mozilla.gecko.annotation.WrapForJNI; + +// A subset of the class AudioInfo in dom/media/MediaInfo.h +@WrapForJNI +public final class GeckoAudioInfo { + public final byte[] codecSpecificData; + public final int rate; + public final int channels; + public final int bitDepth; + public final int profile; + public final long duration; + public final String mimeType; + + public GeckoAudioInfo( + final int rate, + final int channels, + final int bitDepth, + final int profile, + final long duration, + final String mimeType, + final byte[] codecSpecificData) { + this.rate = rate; + this.channels = channels; + this.bitDepth = bitDepth; + this.profile = profile; + this.duration = duration; + this.mimeType = mimeType; + this.codecSpecificData = codecSpecificData; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSDemuxerWrapper.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSDemuxerWrapper.java new file mode 100644 index 0000000000..36c714ba72 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSDemuxerWrapper.java @@ -0,0 +1,164 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.util.Log; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.mozilla.gecko.annotation.WrapForJNI; +import org.mozilla.gecko.mozglue.JNIObject; +import org.mozilla.geckoview.BuildConfig; + +public final class GeckoHLSDemuxerWrapper { + private static final String LOGTAG = "GeckoHLSDemuxerWrapper"; + private static final boolean DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + + // NOTE : These TRACK definitions should be synced with Gecko. + public enum TrackType { + UNDEFINED(0), + AUDIO(1), + VIDEO(2), + TEXT(3); + private int mType; + + TrackType(final int type) { + mType = type; + } + + public int value() { + return mType; + } + } + + private BaseHlsPlayer mPlayer = null; + + public static class Callbacks extends JNIObject implements BaseHlsPlayer.DemuxerCallbacks { + @WrapForJNI(calledFrom = "gecko") + Callbacks() {} + + @Override + @WrapForJNI + public native void onInitialized(boolean hasAudio, boolean hasVideo); + + @Override + @WrapForJNI + public native void onError(int errorCode); + + @Override // JNIObject + protected void disposeNative() { + throw new UnsupportedOperationException(); + } + } // Callbacks + + private static void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private BaseHlsPlayer.TrackType getPlayerTrackType(final int trackType) { + if (trackType == TrackType.AUDIO.value()) { + return BaseHlsPlayer.TrackType.AUDIO; + } else if (trackType == TrackType.VIDEO.value()) { + return BaseHlsPlayer.TrackType.VIDEO; + } else if (trackType == TrackType.TEXT.value()) { + return BaseHlsPlayer.TrackType.TEXT; + } + return BaseHlsPlayer.TrackType.UNDEFINED; + } + + @WrapForJNI + public long getBuffered() { + assertTrue(mPlayer != null); + return mPlayer.getBufferedPosition(); + } + + @WrapForJNI(calledFrom = "gecko") + public static GeckoHLSDemuxerWrapper create( + final int id, final BaseHlsPlayer.DemuxerCallbacks callback) { + return new GeckoHLSDemuxerWrapper(id, callback); + } + + @WrapForJNI + public int getNumberOfTracks(final int trackType) { + assertTrue(mPlayer != null); + final int tracks = mPlayer.getNumberOfTracks(getPlayerTrackType(trackType)); + if (DEBUG) Log.d(LOGTAG, "[GetNumberOfTracks] type : " + trackType + ", num = " + tracks); + return tracks; + } + + @WrapForJNI + public GeckoAudioInfo getAudioInfo(final int index) { + assertTrue(mPlayer != null); + if (DEBUG) Log.d(LOGTAG, "[getAudioInfo] formatIndex : " + index); + return mPlayer.getAudioInfo(index); + } + + @WrapForJNI + public GeckoVideoInfo getVideoInfo(final int index) { + assertTrue(mPlayer != null); + if (DEBUG) Log.d(LOGTAG, "[getVideoInfo] formatIndex : " + index); + return mPlayer.getVideoInfo(index); + } + + @WrapForJNI + public boolean seek(final long seekTime) { + // seekTime : microseconds. + assertTrue(mPlayer != null); + if (DEBUG) Log.d(LOGTAG, "seek : " + seekTime + " (Us)"); + return mPlayer.seek(seekTime); + } + + GeckoHLSDemuxerWrapper(final int id, final BaseHlsPlayer.DemuxerCallbacks callback) { + if (DEBUG) Log.d(LOGTAG, "Constructing GeckoHLSDemuxerWrapper ..."); + assertTrue(callback != null); + try { + mPlayer = GeckoPlayerFactory.getPlayer(id); + if (mPlayer != null) { + mPlayer.addDemuxerWrapperCallbackListener(callback); + } + } catch (final Exception e) { + Log.e(LOGTAG, "Constructing GeckoHLSDemuxerWrapper ... error", e); + callback.onError(BaseHlsPlayer.DemuxerError.UNKNOWN.code()); + } + } + + @WrapForJNI + private GeckoHLSSample[] getSamples(final int mediaType, final int number) { + assertTrue(mPlayer != null); + ConcurrentLinkedQueue<GeckoHLSSample> samples = null; + // getA/VSamples will always return a non-null instance. + samples = mPlayer.getSamples(getPlayerTrackType(mediaType), number); + assertTrue(samples.size() <= number); + return samples.toArray(new GeckoHLSSample[samples.size()]); + } + + @WrapForJNI + private long getNextKeyFrameTime() { + assertTrue(mPlayer != null); + return mPlayer.getNextKeyFrameTime(); + } + + @WrapForJNI + private boolean isLiveStream() { + assertTrue(mPlayer != null); + return mPlayer.isLiveStream(); + } + + @WrapForJNI // Called when native object is destroyed. + private void destroy() { + if (DEBUG) Log.d(LOGTAG, "destroy!! Native object is destroyed."); + if (mPlayer != null) { + release(); + } + } + + private void release() { + assertTrue(mPlayer != null); + if (DEBUG) Log.d(LOGTAG, "release BaseHlsPlayer..."); + GeckoPlayerFactory.removePlayer(mPlayer); + mPlayer.release(); + mPlayer = null; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSResourceWrapper.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSResourceWrapper.java new file mode 100644 index 0000000000..c21789fdd0 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSResourceWrapper.java @@ -0,0 +1,119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.util.Log; +import org.mozilla.gecko.annotation.WrapForJNI; +import org.mozilla.gecko.mozglue.JNIObject; +import org.mozilla.geckoview.BuildConfig; + +public class GeckoHLSResourceWrapper { + private static final String LOGTAG = "GeckoHLSResourceWrapper"; + private static final boolean DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + private BaseHlsPlayer mPlayer = null; + private boolean mDestroy = false; + + public static class Callbacks extends JNIObject implements BaseHlsPlayer.ResourceCallbacks { + @WrapForJNI(calledFrom = "gecko") + Callbacks() {} + + @Override + @WrapForJNI + public native void onLoad(String mediaUrl); + + @Override + @WrapForJNI + public native void onDataArrived(); + + @Override + @WrapForJNI + public native void onError(int errorCode); + + @Override // JNIObject + protected void disposeNative() { + throw new UnsupportedOperationException(); + } + } // Callbacks + + private GeckoHLSResourceWrapper( + final String url, final BaseHlsPlayer.ResourceCallbacks callback) { + if (DEBUG) Log.d(LOGTAG, "GeckoHLSResourceWrapper created with url = " + url); + assertTrue(callback != null); + + mPlayer = GeckoPlayerFactory.getPlayer(); + try { + mPlayer.init(url, callback); + } catch (final Exception e) { + Log.e(LOGTAG, "Failed to create GeckoHlsResourceWrapper !", e); + callback.onError(BaseHlsPlayer.ResourceError.UNKNOWN.code()); + } + } + + @WrapForJNI(calledFrom = "gecko") + public static GeckoHLSResourceWrapper create( + final String url, final BaseHlsPlayer.ResourceCallbacks callback) { + return new GeckoHLSResourceWrapper(url, callback); + } + + @WrapForJNI(calledFrom = "gecko") + public int getPlayerId() { + // GeckoHLSResourceWrapper should always be created before others + assertTrue(!mDestroy); + assertTrue(mPlayer != null); + return mPlayer.getId(); + } + + @WrapForJNI(calledFrom = "gecko") + public void suspend() { + if (DEBUG) Log.d(LOGTAG, "GeckoHLSResourceWrapper suspend"); + if (mPlayer != null) { + mPlayer.suspend(); + } + } + + @WrapForJNI(calledFrom = "gecko") + public void resume() { + if (DEBUG) Log.d(LOGTAG, "GeckoHLSResourceWrapper resume"); + if (mPlayer != null) { + mPlayer.resume(); + } + } + + @WrapForJNI(calledFrom = "gecko") + public void play() { + if (DEBUG) Log.d(LOGTAG, "GeckoHLSResourceWrapper mediaelement played"); + if (mPlayer != null) { + mPlayer.play(); + } + } + + @WrapForJNI(calledFrom = "gecko") + public void pause() { + if (DEBUG) Log.d(LOGTAG, "GeckoHLSResourceWrapper mediaelement paused"); + if (mPlayer != null) { + mPlayer.pause(); + } + } + + private static void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + @WrapForJNI // Called when native object is mDestroy. + private void destroy() { + if (DEBUG) Log.d(LOGTAG, "destroy!! Native object is destroyed."); + if (mDestroy) { + return; + } + mDestroy = true; + if (mPlayer != null) { + GeckoPlayerFactory.removePlayer(mPlayer); + mPlayer.release(); + mPlayer = null; + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSSample.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSSample.java new file mode 100644 index 0000000000..d2ab76a13d --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHLSSample.java @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.mozilla.gecko.annotation.WrapForJNI; + +public final class GeckoHLSSample { + public static final GeckoHLSSample EOS; + + static { + final BufferInfo eosInfo = new BufferInfo(); + eosInfo.set(0, 0, Long.MIN_VALUE, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + EOS = new GeckoHLSSample(null, eosInfo, null, 0); + } + + // Indicate the index of format which is used by this sample. + @WrapForJNI public final int formatIndex; + + @WrapForJNI public long duration; + + @WrapForJNI public final BufferInfo info; + + @WrapForJNI public final CryptoInfo cryptoInfo; + + private ByteBuffer mBuffer = null; + + @WrapForJNI + public void writeToByteBuffer(final ByteBuffer dest) throws IOException { + if (mBuffer != null && dest != null && info.size > 0) { + dest.put(mBuffer); + } + } + + @WrapForJNI + public boolean isEOS() { + return (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + } + + @WrapForJNI + public boolean isKeyFrame() { + return (info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0; + } + + public static GeckoHLSSample create( + final ByteBuffer src, + final BufferInfo info, + final CryptoInfo cryptoInfo, + final int formatIndex) { + return new GeckoHLSSample(src, info, cryptoInfo, formatIndex); + } + + private GeckoHLSSample( + final ByteBuffer buffer, + final BufferInfo info, + final CryptoInfo cryptoInfo, + final int formatIndex) { + this.formatIndex = formatIndex; + duration = Long.MAX_VALUE; + this.mBuffer = buffer; + this.info = info; + this.cryptoInfo = cryptoInfo; + } + + @Override + public String toString() { + if (isEOS()) { + return "EOS GeckoHLSSample"; + } + + final StringBuilder str = new StringBuilder(); + str.append("{ info=") + .append("{ offset=") + .append(info.offset) + .append(", size=") + .append(info.size) + .append(", pts=") + .append(info.presentationTimeUs) + .append(", duration=") + .append(duration) + .append(", flags=") + .append(Integer.toHexString(info.flags)) + .append(" }") + .append(" }"); + return str.toString(); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsAudioRenderer.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsAudioRenderer.java new file mode 100644 index 0000000000..d60f7c1ccd --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsAudioRenderer.java @@ -0,0 +1,167 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import android.util.Log; +import java.nio.ByteBuffer; +import java.util.List; +import org.mozilla.geckoview.BuildConfig; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; +import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; +import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecInfo; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecUtil; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.MimeTypes; + +public class GeckoHlsAudioRenderer extends GeckoHlsRendererBase { + public GeckoHlsAudioRenderer(final GeckoHlsPlayer.ComponentEventDispatcher eventDispatcher) { + super(C.TRACK_TYPE_AUDIO, eventDispatcher); + LOGTAG = getClass().getSimpleName(); + DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + } + + @Override + public final int supportsFormat(final Format format) { + /* + * FORMAT_EXCEEDS_CAPABILITIES : The Renderer is capable of rendering + * formats with the same mime type, but + * the properties of the format exceed + * the renderer's capability. + * FORMAT_UNSUPPORTED_SUBTYPE : The Renderer is a general purpose + * renderer for formats of the same + * top-level type, but is not capable of + * rendering the format or any other format + * with the same mime type because the + * sub-type is not supported. + * FORMAT_UNSUPPORTED_TYPE : The Renderer is not capable of rendering + * the format, either because it does not support + * the format's top-level type, or because it's + * a specialized renderer for a different mime type. + * ADAPTIVE_NOT_SEAMLESS : The Renderer can adapt between formats, + * but may suffer a brief discontinuity (~50-100ms) + * when adaptation occurs. + */ + final String mimeType = format.sampleMimeType; + if (!MimeTypes.isAudio(mimeType)) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + } + List<MediaCodecInfo> decoderInfos = null; + try { + final MediaCodecSelector mediaCodecSelector = MediaCodecSelector.DEFAULT; + decoderInfos = mediaCodecSelector.getDecoderInfos(mimeType, false, false); + } catch (final MediaCodecUtil.DecoderQueryException e) { + Log.e(LOGTAG, e.getMessage()); + } + if (decoderInfos == null || decoderInfos.isEmpty()) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE); + } + final MediaCodecInfo info = decoderInfos.get(0); + /* + * Note : If the code can make it to this place, ExoPlayer assumes + * support for unknown sampleRate and channelCount when + * SDK version is less than 21, otherwise, further check is needed + * if there's no sampleRate/channelCount in format. + */ + final boolean decoderCapable = + ((format.sampleRate == Format.NO_VALUE + || info.isAudioSampleRateSupportedV21(format.sampleRate)) + && (format.channelCount == Format.NO_VALUE + || info.isAudioChannelCountSupportedV21(format.channelCount))); + return RendererCapabilities.create( + decoderCapable ? FORMAT_HANDLED : FORMAT_EXCEEDS_CAPABILITIES, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED); + } + + @Override + protected final void createInputBuffer() { + // We're not able to estimate the size for audio from format. So we rely + // on the dynamic allocation mechanism provided in DecoderInputBuffer. + mInputBuffer = null; + } + + @Override + protected void resetRenderer() { + mInputBuffer = null; + mInitialized = false; + } + + @Override + protected void handleReconfiguration(final DecoderInputBuffer bufferForRead) { + // Do nothing + } + + @Override + protected void handleFormatRead(final DecoderInputBuffer bufferForRead) + throws ExoPlaybackException { + onInputFormatChanged(mFormatHolder.format); + } + + @Override + protected void handleEndOfStream(final DecoderInputBuffer bufferForRead) { + mInputStreamEnded = true; + mDemuxedInputSamples.offer(GeckoHLSSample.EOS); + } + + @Override + protected void handleSamplePreparation(final DecoderInputBuffer bufferForRead) { + final int size = bufferForRead.data.limit(); + final byte[] realData = new byte[size]; + bufferForRead.data.get(realData, 0, size); + final ByteBuffer buffer = ByteBuffer.wrap(realData); + mInputBuffer = bufferForRead.data; + mInputBuffer.clear(); + + final CryptoInfo cryptoInfo = + bufferForRead.isEncrypted() ? bufferForRead.cryptoInfo.getFrameworkCryptoInfoV16() : null; + final BufferInfo bufferInfo = new BufferInfo(); + // Flags in DecoderInputBuffer are synced with MediaCodec Buffer flags. + int flags = 0; + flags |= bufferForRead.isKeyFrame() ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0; + flags |= bufferForRead.isEndOfStream() ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0; + bufferInfo.set(0, size, bufferForRead.timeUs, flags); + + assertTrue(mFormats.size() >= 0); + // We add a new format in the list once format changes, so the formatIndex + // should indicate to the last(latest) format. + final GeckoHLSSample sample = + GeckoHLSSample.create(buffer, bufferInfo, cryptoInfo, mFormats.size() - 1); + + mDemuxedInputSamples.offer(sample); + + if (BuildConfig.DEBUG_BUILD) { + Log.d( + LOGTAG, + "Demuxed sample PTS : " + + sample.info.presentationTimeUs + + ", duration :" + + sample.duration + + ", formatIndex(" + + sample.formatIndex + + "), queue size : " + + mDemuxedInputSamples.size()); + } + } + + @Override + protected boolean clearInputSamplesQueue() { + if (DEBUG) { + Log.d(LOGTAG, "clearInputSamplesQueue"); + } + mDemuxedInputSamples.clear(); + return true; + } + + @Override + protected void notifyPlayerInputFormatChanged(final Format newFormat) { + mPlayerEventDispatcher.onAudioInputFormatChanged(newFormat); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsPlayer.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsPlayer.java new file mode 100644 index 0000000000..4fe5064072 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsPlayer.java @@ -0,0 +1,1107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.content.Context; +import android.net.Uri; +import android.os.Handler; +import android.os.HandlerThread; +import android.util.Log; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.FutureTask; +import java.util.concurrent.atomic.AtomicInteger; +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.annotation.ReflectionTarget; +import org.mozilla.geckoview.BuildConfig; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.DefaultLoadControl; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlayer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; +import org.mozilla.thirdparty.com.google.android.exoplayer2.PlaybackParameters; +import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Timeline; +import org.mozilla.thirdparty.com.google.android.exoplayer2.source.MediaSource; +import org.mozilla.thirdparty.com.google.android.exoplayer2.source.MediaSourceEventListener; +import org.mozilla.thirdparty.com.google.android.exoplayer2.source.TrackGroup; +import org.mozilla.thirdparty.com.google.android.exoplayer2.source.TrackGroupArray; +import org.mozilla.thirdparty.com.google.android.exoplayer2.source.hls.HlsMediaSource; +import org.mozilla.thirdparty.com.google.android.exoplayer2.trackselection.AdaptiveTrackSelection; +import org.mozilla.thirdparty.com.google.android.exoplayer2.trackselection.DefaultTrackSelector; +import org.mozilla.thirdparty.com.google.android.exoplayer2.trackselection.MappingTrackSelector.MappedTrackInfo; +import org.mozilla.thirdparty.com.google.android.exoplayer2.trackselection.TrackSelection; +import org.mozilla.thirdparty.com.google.android.exoplayer2.trackselection.TrackSelectionArray; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.DefaultAllocator; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.DefaultBandwidthMeter; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.DefaultDataSourceFactory; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.DefaultHttpDataSource; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.DefaultHttpDataSourceFactory; +import org.mozilla.thirdparty.com.google.android.exoplayer2.upstream.HttpDataSource; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.MimeTypes; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Util; + +@ReflectionTarget +public class GeckoHlsPlayer implements BaseHlsPlayer, ExoPlayer.EventListener { + private static final String LOGTAG = "GeckoHlsPlayer"; + private static final DefaultBandwidthMeter BANDWIDTH_METER = + new DefaultBandwidthMeter.Builder(null).build(); + private static final int MAX_TIMELINE_ITEM_LINES = 3; + private static final boolean DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + + private static final AtomicInteger sPlayerId = new AtomicInteger(0); + /* + * Because we treat GeckoHlsPlayer as a source data provider. + * It will be created and initialized with a URL by HLSResource in + * Gecko media pipleine (in cpp). Once HLSDemuxer is created later, we + * need to bridge this HLSResource to the created demuxer. And they share + * the same GeckoHlsPlayer. + * mPlayerId is a token used for Gecko media pipeline to obtain corresponding player. + */ + private final int mPlayerId; + // Accessed only in GeckoHlsPlayerThread. + private boolean mExoplayerSuspended = false; + + private static final int DEFAULT_MIN_BUFFER_MS = 5 * 1000; + private static final int DEFAULT_MAX_BUFFER_MS = 10 * 1000; + + private enum MediaDecoderPlayState { + PLAY_STATE_PREPARING, + PLAY_STATE_PAUSED, + PLAY_STATE_PLAYING + } + + // Default value is PLAY_STATE_PREPARING and it will be set to PLAY_STATE_PLAYING + // once HTMLMediaElement calls PlayInternal(). + // Accessed only in GeckoHlsPlayerThread. + private MediaDecoderPlayState mMediaDecoderPlayState = MediaDecoderPlayState.PLAY_STATE_PREPARING; + + private Handler mMainHandler; + private HandlerThread mThread; + private ExoPlayer mPlayer; + private GeckoHlsRendererBase[] mRenderers; + private DefaultTrackSelector mTrackSelector; + private MediaSource mMediaSource; + private SourceEventListener mSourceEventListener; + private ComponentListener mComponentListener; + private ComponentEventDispatcher mComponentEventDispatcher; + + private volatile boolean mIsTimelineStatic = false; + private long mDurationUs; + + private GeckoHlsVideoRenderer mVRenderer = null; + private GeckoHlsAudioRenderer mARenderer = null; + + // Able to control if we only want V/A/V+A tracks from bitstream. + private class RendererController { + private final boolean mEnableV; + private final boolean mEnableA; + + RendererController(final boolean enableVideoRenderer, final boolean enableAudioRenderer) { + this.mEnableV = enableVideoRenderer; + this.mEnableA = enableAudioRenderer; + } + + boolean isVideoRendererEnabled() { + return mEnableV; + } + + boolean isAudioRendererEnabled() { + return mEnableA; + } + } + + private RendererController mRendererController = new RendererController(true, true); + + // Provide statistical information of tracks. + private class HlsMediaTracksInfo { + private int mNumVideoTracks = 0; + private int mNumAudioTracks = 0; + private boolean mVideoInfoUpdated = false; + private boolean mAudioInfoUpdated = false; + private boolean mVideoDataArrived = false; + private boolean mAudioDataArrived = false; + + HlsMediaTracksInfo() {} + + public void reset() { + mNumVideoTracks = 0; + mNumAudioTracks = 0; + mVideoInfoUpdated = false; + mAudioInfoUpdated = false; + mVideoDataArrived = false; + mAudioDataArrived = false; + } + + public void updateNumOfVideoTracks(final int numOfTracks) { + mNumVideoTracks = numOfTracks; + } + + public void updateNumOfAudioTracks(final int numOfTracks) { + mNumAudioTracks = numOfTracks; + } + + public boolean hasVideo() { + return mNumVideoTracks > 0; + } + + public boolean hasAudio() { + return mNumAudioTracks > 0; + } + + public int getNumOfVideoTracks() { + return mNumVideoTracks; + } + + public int getNumOfAudioTracks() { + return mNumAudioTracks; + } + + public void onVideoInfoUpdated() { + mVideoInfoUpdated = true; + } + + public void onAudioInfoUpdated() { + mAudioInfoUpdated = true; + } + + public void onDataArrived(final int trackType) { + if (trackType == C.TRACK_TYPE_VIDEO) { + mVideoDataArrived = true; + } else if (trackType == C.TRACK_TYPE_AUDIO) { + mAudioDataArrived = true; + } + } + + public boolean videoReady() { + return !hasVideo() || (mVideoInfoUpdated && mVideoDataArrived); + } + + public boolean audioReady() { + return !hasAudio() || (mAudioInfoUpdated && mAudioDataArrived); + } + } + + private HlsMediaTracksInfo mTracksInfo = new HlsMediaTracksInfo(); + + // Used only in GeckoHlsPlayerThread. + private boolean mIsPlayerInitDone = false; + private boolean mIsDemuxerInitDone = false; + private BaseHlsPlayer.DemuxerCallbacks mDemuxerCallbacks; + private BaseHlsPlayer.ResourceCallbacks mResourceCallbacks; + + private boolean mReleasing = false; // Used only in Gecko Main thread. + + private static void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + protected void checkInitDone() { + if (mIsDemuxerInitDone) { + return; + } + assertTrue(mDemuxerCallbacks != null); + + if (DEBUG) { + Log.d( + LOGTAG, + "[checkInitDone] VReady:" + + mTracksInfo.videoReady() + + ",AReady:" + + mTracksInfo.audioReady() + + ",hasV:" + + mTracksInfo.hasVideo() + + ",hasA:" + + mTracksInfo.hasAudio()); + } + if (mTracksInfo.videoReady() && mTracksInfo.audioReady()) { + if (mDemuxerCallbacks != null) { + mDemuxerCallbacks.onInitialized(mTracksInfo.hasAudio(), mTracksInfo.hasVideo()); + } + mIsDemuxerInitDone = true; + } + } + + private final class SourceEventListener implements MediaSourceEventListener { + public void onLoadStarted( + final int windowIndex, + final MediaSource.MediaPeriodId mediaPeriodId, + final LoadEventInfo loadEventInfo, + final MediaLoadData mediaLoadData) { + assertTrue(isPlayerThread()); + + synchronized (GeckoHlsPlayer.this) { + if (mediaLoadData.dataType != C.DATA_TYPE_MEDIA) { + // Don't report non-media URLs. + return; + } + if (mResourceCallbacks == null || loadEventInfo.uri == null || mReleasing) { + return; + } + + if (DEBUG) { + Log.d(LOGTAG, "on-load: url=" + loadEventInfo.uri); + } + mResourceCallbacks.onLoad(loadEventInfo.uri.toString()); + } + } + } + + public final class ComponentEventDispatcher { + // Called from GeckoHls{Audio,Video}Renderer/ExoPlayer internal playback thread + // or GeckoHlsPlayerThread. + public void onDataArrived(final int trackType) { + assertTrue(mComponentListener != null); + + if (mComponentListener != null) { + runOnPlayerThread(() -> mComponentListener.onDataArrived(trackType)); + } + } + + // Called from GeckoHls{Audio,Video}Renderer internal playback thread. + public void onVideoInputFormatChanged(final Format format) { + assertTrue(mComponentListener != null); + + if (mComponentListener != null) { + runOnPlayerThread(() -> mComponentListener.onVideoInputFormatChanged(format)); + } + } + + // Called from GeckoHls{Audio,Video}Renderer internal playback thread. + public void onAudioInputFormatChanged(final Format format) { + assertTrue(mComponentListener != null); + + if (mComponentListener != null) { + runOnPlayerThread(() -> mComponentListener.onAudioInputFormatChanged(format)); + } + } + } + + public final class ComponentListener { + + // General purpose implementation + // Called on GeckoHlsPlayerThread + public void onDataArrived(final int trackType) { + assertTrue(isPlayerThread()); + + synchronized (GeckoHlsPlayer.this) { + if (DEBUG) { + Log.d(LOGTAG, "[CB][onDataArrived] id " + mPlayerId); + } + if (!mIsPlayerInitDone) { + return; + } + + mTracksInfo.onDataArrived(trackType); + if (!mReleasing) { + mResourceCallbacks.onDataArrived(); + } + checkInitDone(); + } + } + + // Called on GeckoHlsPlayerThread + public void onVideoInputFormatChanged(final Format format) { + assertTrue(isPlayerThread()); + + synchronized (GeckoHlsPlayer.this) { + if (DEBUG) { + Log.d(LOGTAG, "[CB] onVideoInputFormatChanged [" + format + "]"); + Log.d( + LOGTAG, + "[CB] SampleMIMEType [" + + format.sampleMimeType + + "], ContainerMIMEType [" + + format.containerMimeType + + "], id : " + + mPlayerId); + } + if (!mIsPlayerInitDone) { + return; + } + mTracksInfo.onVideoInfoUpdated(); + checkInitDone(); + } + } + + // Called on GeckoHlsPlayerThread + public void onAudioInputFormatChanged(final Format format) { + assertTrue(isPlayerThread()); + + synchronized (GeckoHlsPlayer.this) { + if (DEBUG) { + Log.d(LOGTAG, "[CB] onAudioInputFormatChanged [" + format + "], mPlayerId :" + mPlayerId); + } + if (!mIsPlayerInitDone) { + return; + } + mTracksInfo.onAudioInfoUpdated(); + checkInitDone(); + } + } + } + + private HlsMediaSource.Factory buildDataSourceFactory( + final Context ctx, final DefaultBandwidthMeter bandwidthMeter) { + return new HlsMediaSource.Factory( + new DefaultDataSourceFactory( + ctx, bandwidthMeter, buildHttpDataSourceFactory(bandwidthMeter))); + } + + private HttpDataSource.Factory buildHttpDataSourceFactory( + final DefaultBandwidthMeter bandwidthMeter) { + return new DefaultHttpDataSourceFactory( + BuildConfig.USER_AGENT_GECKOVIEW_MOBILE, + bandwidthMeter /* listener */, + DefaultHttpDataSource.DEFAULT_CONNECT_TIMEOUT_MILLIS, + DefaultHttpDataSource.DEFAULT_READ_TIMEOUT_MILLIS, + true /* allowCrossProtocolRedirects */); + } + + private long getDuration() { + return awaitPlayerThread( + () -> { + long duration = 0L; + // Value returned by getDuration() is in milliseconds. + if (mPlayer != null && !isLiveStream()) { + duration = Math.max(0L, mPlayer.getDuration() * 1000L); + } + if (DEBUG) { + Log.d(LOGTAG, "getDuration : " + duration + "(Us)"); + } + return duration; + }); + } + + // To make sure that each player has a unique id, GeckoHlsPlayer should be + // created only from synchronized APIs in GeckoPlayerFactory. + public GeckoHlsPlayer() { + mPlayerId = sPlayerId.incrementAndGet(); + if (DEBUG) { + Log.d(LOGTAG, " construct player with id(" + mPlayerId + ")"); + } + } + + // Should be only called by GeckoPlayerFactory and GeckoHLSResourceWrapper. + // The mPlayerId is used to make sure that the same GeckoHlsPlayer is used by + // corresponding HLSResource and HLSDemuxer for each media playback. + // Called on Gecko's main thread + @Override + public int getId() { + return mPlayerId; + } + + // Called on Gecko's main thread + @Override + public synchronized void addDemuxerWrapperCallbackListener( + final BaseHlsPlayer.DemuxerCallbacks callback) { + if (DEBUG) { + Log.d(LOGTAG, " addDemuxerWrapperCallbackListener ..."); + } + mDemuxerCallbacks = callback; + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public synchronized void onLoadingChanged(final boolean isLoading) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.d(LOGTAG, "loading [" + isLoading + "]"); + } + if (!isLoading) { + if (mMediaDecoderPlayState != MediaDecoderPlayState.PLAY_STATE_PLAYING) { + suspendExoplayer(); + } + // To update buffered position. + mComponentEventDispatcher.onDataArrived(C.TRACK_TYPE_DEFAULT); + } + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public synchronized void onPlayerStateChanged(final boolean playWhenReady, final int state) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.d(LOGTAG, "state [" + playWhenReady + ", " + getStateString(state) + "]"); + } + if (state == ExoPlayer.STATE_READY + && !mExoplayerSuspended + && mMediaDecoderPlayState == MediaDecoderPlayState.PLAY_STATE_PLAYING) { + resumeExoplayer(); + } + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public void onPositionDiscontinuity(final int reason) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.d(LOGTAG, "positionDiscontinuity: reason=" + reason); + } + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public void onPlaybackParametersChanged(final PlaybackParameters playbackParameters) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.d( + LOGTAG, + "playbackParameters " + + String.format( + "[speed=%.2f, pitch=%.2f]", playbackParameters.speed, playbackParameters.pitch)); + } + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public synchronized void onPlayerError(final ExoPlaybackException e) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.e(LOGTAG, "playerFailed", e); + } + mIsPlayerInitDone = false; + if (mReleasing) { + return; + } + if (mResourceCallbacks != null) { + mResourceCallbacks.onError(ResourceError.PLAYER.code()); + } + if (mDemuxerCallbacks != null) { + mDemuxerCallbacks.onError(DemuxerError.PLAYER.code()); + } + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public synchronized void onTracksChanged( + final TrackGroupArray ignored, final TrackSelectionArray trackSelections) { + assertTrue(isPlayerThread()); + + if (DEBUG) { + Log.d(LOGTAG, "onTracksChanged : TGA[" + ignored + "], TSA[" + trackSelections + "]"); + + final MappedTrackInfo mappedTrackInfo = mTrackSelector.getCurrentMappedTrackInfo(); + if (mappedTrackInfo == null) { + Log.d(LOGTAG, "Tracks []"); + return; + } + Log.d(LOGTAG, "Tracks ["); + // Log tracks associated to renderers. + for (int rendererIndex = 0; rendererIndex < mappedTrackInfo.length; rendererIndex++) { + final TrackGroupArray rendererTrackGroups = mappedTrackInfo.getTrackGroups(rendererIndex); + final TrackSelection trackSelection = trackSelections.get(rendererIndex); + if (rendererTrackGroups.length > 0) { + Log.d(LOGTAG, " Renderer:" + rendererIndex + " ["); + for (int groupIndex = 0; groupIndex < rendererTrackGroups.length; groupIndex++) { + final TrackGroup trackGroup = rendererTrackGroups.get(groupIndex); + final String adaptiveSupport = + getAdaptiveSupportString( + trackGroup.length, + mappedTrackInfo.getAdaptiveSupport(rendererIndex, groupIndex, false)); + Log.d( + LOGTAG, + " Group:" + groupIndex + ", adaptive_supported=" + adaptiveSupport + " ["); + for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { + final String status = getTrackStatusString(trackSelection, trackGroup, trackIndex); + final String formatSupport = + getFormatSupportString( + mappedTrackInfo.getTrackFormatSupport(rendererIndex, groupIndex, trackIndex)); + Log.d( + LOGTAG, + " " + + status + + " Track:" + + trackIndex + + ", " + + Format.toLogString(trackGroup.getFormat(trackIndex)) + + ", supported=" + + formatSupport); + } + Log.d(LOGTAG, " ]"); + } + Log.d(LOGTAG, " ]"); + } + } + // Log tracks not associated with a renderer. + final TrackGroupArray unassociatedTrackGroups = mappedTrackInfo.getUnassociatedTrackGroups(); + if (unassociatedTrackGroups.length > 0) { + Log.d(LOGTAG, " Renderer:None ["); + for (int groupIndex = 0; groupIndex < unassociatedTrackGroups.length; groupIndex++) { + Log.d(LOGTAG, " Group:" + groupIndex + " ["); + final TrackGroup trackGroup = unassociatedTrackGroups.get(groupIndex); + for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { + final String status = getTrackStatusString(false); + final String formatSupport = + getFormatSupportString(RendererCapabilities.FORMAT_UNSUPPORTED_TYPE); + Log.d( + LOGTAG, + " " + + status + + " Track:" + + trackIndex + + ", " + + Format.toLogString(trackGroup.getFormat(trackIndex)) + + ", supported=" + + formatSupport); + } + Log.d(LOGTAG, " ]"); + } + Log.d(LOGTAG, " ]"); + } + Log.d(LOGTAG, "]"); + } + mTracksInfo.reset(); + int numVideoTracks = 0; + int numAudioTracks = 0; + for (int j = 0; j < ignored.length; j++) { + final TrackGroup tg = ignored.get(j); + for (int i = 0; i < tg.length; i++) { + final Format fmt = tg.getFormat(i); + if (fmt.sampleMimeType != null) { + if (mRendererController.isVideoRendererEnabled() + && fmt.sampleMimeType.startsWith(new String("video"))) { + numVideoTracks++; + } else if (mRendererController.isAudioRendererEnabled() + && fmt.sampleMimeType.startsWith(new String("audio"))) { + numAudioTracks++; + } + } + } + } + mTracksInfo.updateNumOfVideoTracks(numVideoTracks); + mTracksInfo.updateNumOfAudioTracks(numAudioTracks); + } + + // Called on GeckoHlsPlayerThread from ExoPlayer + @Override + public synchronized void onTimelineChanged(final Timeline timeline, final int reason) { + assertTrue(isPlayerThread()); + + // For now, we use the interface ExoPlayer.getDuration() for gecko, + // so here we create local variable 'window' & 'peroid' to obtain + // the dynamic duration. + // See. + // http://google.github.io/ExoPlayer/doc/reference/com/google/android/exoplayer2/Timeline.html + // for further information. + final Timeline.Window window = new Timeline.Window(); + mIsTimelineStatic = + !timeline.isEmpty() && !timeline.getWindow(timeline.getWindowCount() - 1, window).isDynamic; + + final int periodCount = timeline.getPeriodCount(); + final int windowCount = timeline.getWindowCount(); + if (DEBUG) { + Log.d(LOGTAG, "sourceInfo [periodCount=" + periodCount + ", windowCount=" + windowCount); + } + final Timeline.Period period = new Timeline.Period(); + for (int i = 0; i < Math.min(periodCount, MAX_TIMELINE_ITEM_LINES); i++) { + timeline.getPeriod(i, period); + if (mDurationUs < period.getDurationUs()) { + mDurationUs = period.getDurationUs(); + } + } + for (int i = 0; i < Math.min(windowCount, MAX_TIMELINE_ITEM_LINES); i++) { + timeline.getWindow(i, window); + if (mDurationUs < window.getDurationUs()) { + mDurationUs = window.getDurationUs(); + } + } + // TODO : Need to check if the duration from play.getDuration is different + // with the one calculated from multi-timelines/windows. + if (DEBUG) { + Log.d( + LOGTAG, + "Media duration (from Timeline) = " + + mDurationUs + + "(us)" + + " player.getDuration() = " + + mPlayer.getDuration() + + "(ms)"); + } + } + + private static String getStateString(final int state) { + switch (state) { + case ExoPlayer.STATE_BUFFERING: + return "B"; + case ExoPlayer.STATE_ENDED: + return "E"; + case ExoPlayer.STATE_IDLE: + return "I"; + case ExoPlayer.STATE_READY: + return "R"; + default: + return "?"; + } + } + + private static String getFormatSupportString(final int formatSupport) { + switch (formatSupport) { + case RendererCapabilities.FORMAT_HANDLED: + return "YES"; + case RendererCapabilities.FORMAT_EXCEEDS_CAPABILITIES: + return "NO_EXCEEDS_CAPABILITIES"; + case RendererCapabilities.FORMAT_UNSUPPORTED_SUBTYPE: + return "NO_UNSUPPORTED_TYPE"; + case RendererCapabilities.FORMAT_UNSUPPORTED_TYPE: + return "NO"; + default: + return "?"; + } + } + + private static String getAdaptiveSupportString(final int trackCount, final int adaptiveSupport) { + if (trackCount < 2) { + return "N/A"; + } + switch (adaptiveSupport) { + case RendererCapabilities.ADAPTIVE_SEAMLESS: + return "YES"; + case RendererCapabilities.ADAPTIVE_NOT_SEAMLESS: + return "YES_NOT_SEAMLESS"; + case RendererCapabilities.ADAPTIVE_NOT_SUPPORTED: + return "NO"; + default: + return "?"; + } + } + + private static String getTrackStatusString( + final TrackSelection selection, final TrackGroup group, final int trackIndex) { + return getTrackStatusString( + selection != null + && selection.getTrackGroup() == group + && selection.indexOf(trackIndex) != C.INDEX_UNSET); + } + + private static String getTrackStatusString(final boolean enabled) { + return enabled ? "[X]" : "[ ]"; + } + + // Called on GeckoHlsPlayerThread + private void createExoPlayer(final String url) { + assertTrue(isPlayerThread()); + + final Context ctx = GeckoAppShell.getApplicationContext(); + mComponentListener = new ComponentListener(); + mComponentEventDispatcher = new ComponentEventDispatcher(); + mDurationUs = 0; + + // Prepare trackSelector + final TrackSelection.Factory videoTrackSelectionFactory = + new AdaptiveTrackSelection.Factory(BANDWIDTH_METER); + mTrackSelector = new DefaultTrackSelector(videoTrackSelectionFactory); + + // Prepare customized renderer + mRenderers = new GeckoHlsRendererBase[2]; + mVRenderer = new GeckoHlsVideoRenderer(mComponentEventDispatcher); + mARenderer = new GeckoHlsAudioRenderer(mComponentEventDispatcher); + mRenderers[0] = mVRenderer; + mRenderers[1] = mARenderer; + + final DefaultLoadControl dlc = + new DefaultLoadControl.Builder() + .setAllocator(new DefaultAllocator(true, C.DEFAULT_BUFFER_SEGMENT_SIZE)) + .setBufferDurationsMs( + DEFAULT_MIN_BUFFER_MS, + DEFAULT_MAX_BUFFER_MS, + DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_MS, + DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS) + .createDefaultLoadControl(); + // Create ExoPlayer instance with specific components. + mPlayer = + new ExoPlayer.Builder(ctx, mRenderers) + .setTrackSelector(mTrackSelector) + .setLoadControl(dlc) + .build(); + mPlayer.addListener(this); + + final Uri uri = Uri.parse(url); + mMediaSource = buildDataSourceFactory(ctx, BANDWIDTH_METER).createMediaSource(uri); + mSourceEventListener = new SourceEventListener(); + mMediaSource.addEventListener(mMainHandler, mSourceEventListener); + if (DEBUG) { + Log.d( + LOGTAG, + "Uri is " + uri + ", ContentType is " + Util.inferContentType(uri.getLastPathSegment())); + } + mPlayer.setPlayWhenReady(false); + mPlayer.prepare(mMediaSource); + mIsPlayerInitDone = true; + } + + // ======================================================================= + // API for GeckoHLSResourceWrapper + // ======================================================================= + // Called on Gecko Main Thread + @Override + public synchronized void init(final String url, final BaseHlsPlayer.ResourceCallbacks callback) { + if (DEBUG) { + Log.d(LOGTAG, " init"); + } + assertTrue(callback != null); + assertTrue(!mIsPlayerInitDone); + + mThread = new HandlerThread("GeckoHlsPlayerThread"); + mThread.start(); + mMainHandler = new Handler(mThread.getLooper()); + + mMainHandler.post( + () -> { + mResourceCallbacks = callback; + createExoPlayer(url); + }); + } + + // Called on MDSM's TaskQueue + @Override + public boolean isLiveStream() { + return !mIsTimelineStatic; + } + + // ======================================================================= + // API for GeckoHLSDemuxerWrapper + // ======================================================================= + // Called on HLSDemuxer's TaskQueue + @Override + public synchronized ConcurrentLinkedQueue<GeckoHLSSample> getSamples( + final TrackType trackType, final int number) { + if (trackType == TrackType.VIDEO) { + return mVRenderer != null + ? mVRenderer.getQueuedSamples(number) + : new ConcurrentLinkedQueue<GeckoHLSSample>(); + } else if (trackType == TrackType.AUDIO) { + return mARenderer != null + ? mARenderer.getQueuedSamples(number) + : new ConcurrentLinkedQueue<GeckoHLSSample>(); + } else { + return new ConcurrentLinkedQueue<GeckoHLSSample>(); + } + } + + // Called on MFR's TaskQueue + @Override + public long getBufferedPosition() { + return awaitPlayerThread( + () -> { + // Value returned by getBufferedPosition() is in milliseconds. + final long bufferedPos = + mPlayer == null ? 0L : Math.max(0L, mPlayer.getBufferedPosition() * 1000L); + if (DEBUG) { + Log.d(LOGTAG, "getBufferedPosition : " + bufferedPos + "(Us)"); + } + return bufferedPos; + }); + } + + // Called on MFR's TaskQueue + @Override + public synchronized int getNumberOfTracks(final TrackType trackType) { + if (DEBUG) { + Log.d(LOGTAG, "getNumberOfTracks : type " + trackType); + } + if (trackType == TrackType.VIDEO) { + return mTracksInfo.getNumOfVideoTracks(); + } else if (trackType == TrackType.AUDIO) { + return mTracksInfo.getNumOfAudioTracks(); + } + return 0; + } + + // Called on MFR's TaskQueue + @Override + public GeckoVideoInfo getVideoInfo(final int index) { + final Format fmt; + synchronized (this) { + if (DEBUG) { + Log.d(LOGTAG, "getVideoInfo"); + } + if (mVRenderer == null) { + Log.e(LOGTAG, "no render to get video info from. Index : " + index); + return null; + } + if (!mTracksInfo.hasVideo()) { + return null; + } + fmt = mVRenderer.getFormat(index); + if (fmt == null) { + return null; + } + } + return new GeckoVideoInfo( + fmt.width, + fmt.height, + fmt.width, + fmt.height, + fmt.rotationDegrees, + fmt.stereoMode, + getDuration(), + fmt.sampleMimeType, + null, + null); + } + + // Called on MFR's TaskQueue + @Override + public GeckoAudioInfo getAudioInfo(final int index) { + final Format fmt; + synchronized (this) { + if (DEBUG) { + Log.d(LOGTAG, "getAudioInfo"); + } + if (mARenderer == null) { + Log.e(LOGTAG, "no render to get audio info from. Index : " + index); + return null; + } + if (!mTracksInfo.hasAudio()) { + return null; + } + fmt = mARenderer.getFormat(index); + if (fmt == null) { + return null; + } + } + /* According to https://github.com/google/ExoPlayer/blob + * /d979469659861f7fe1d39d153b90bdff1ab479cc/library/core/src/main + * /java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java#L221-L224, + * if the input audio format is not raw, exoplayer would assure that + * the sample's pcm encoding bitdepth is 16. + * For HLS content, it should always be 16. + */ + assertTrue(!MimeTypes.AUDIO_RAW.equals(fmt.sampleMimeType)); + // For HLS content, csd-0 is enough. + final byte[] csd = fmt.initializationData.isEmpty() ? null : fmt.initializationData.get(0); + return new GeckoAudioInfo( + fmt.sampleRate, fmt.channelCount, 16, 0, getDuration(), fmt.sampleMimeType, csd); + } + + // Called on HLSDemuxer's TaskQueue + @Override + public boolean seek(final long positionUs) { + synchronized (this) { + if (mPlayer == null) { + Log.d(LOGTAG, "Seek operation won't be performed as no player exists!"); + return false; + } + } + return awaitPlayerThread( + () -> { + // Need to temporarily resume Exoplayer to download the chunks for getting the demuxed + // keyframe sample when HTMLMediaElement is paused. Suspend Exoplayer when collecting + // enough + // samples in onLoadingChanged. + if (mExoplayerSuspended) { + resumeExoplayer(); + } + // positionUs : microseconds. + // NOTE : 1) It's not possible to seek media by tracktype via ExoPlayer Interface. + // 2) positionUs is samples PTS from MFR, we need to re-adjust it + // for ExoPlayer by subtracting sample start time. + // 3) Time unit for ExoPlayer.seek() is milliseconds. + try { + // TODO : Gather Timeline Period / Window information to develop + // complete timeline, and seekTime should be inside the duration. + Long startTime = Long.MAX_VALUE; + for (final GeckoHlsRendererBase r : mRenderers) { + if (r == mVRenderer + && mRendererController.isVideoRendererEnabled() + && mTracksInfo.hasVideo() + || r == mARenderer + && mRendererController.isAudioRendererEnabled() + && mTracksInfo.hasAudio()) { + // Find the min value of the start time + startTime = Math.min(startTime, r.getFirstSamplePTS()); + } + } + if (DEBUG) { + Log.d( + LOGTAG, + "seeking : " + + positionUs / 1000 + + " (ms); startTime : " + + startTime / 1000 + + " (ms)"); + } + assertTrue(startTime != Long.MAX_VALUE && startTime != Long.MIN_VALUE); + mPlayer.seekTo(positionUs / 1000 - startTime / 1000); + } catch (final Exception e) { + if (mReleasing) { + return false; + } + if (mDemuxerCallbacks != null) { + mDemuxerCallbacks.onError(DemuxerError.UNKNOWN.code()); + } + return false; + } + return true; + }); + } + + // Called on HLSDemuxer's TaskQueue + @Override + public synchronized long getNextKeyFrameTime() { + return mVRenderer != null ? mVRenderer.getNextKeyFrameTime() : Long.MAX_VALUE; + } + + // Called on Gecko's main thread. + @Override + public synchronized void suspend() { + runOnPlayerThread( + () -> { + if (mExoplayerSuspended) { + return; + } + if (mMediaDecoderPlayState != MediaDecoderPlayState.PLAY_STATE_PLAYING) { + if (DEBUG) { + Log.d(LOGTAG, "suspend player id : " + mPlayerId); + } + suspendExoplayer(); + } + }); + } + + // Called on Gecko's main thread. + @Override + public synchronized void resume() { + runOnPlayerThread( + () -> { + if (!mExoplayerSuspended) { + return; + } + if (mMediaDecoderPlayState == MediaDecoderPlayState.PLAY_STATE_PLAYING) { + if (DEBUG) { + Log.d(LOGTAG, "resume player id : " + mPlayerId); + } + resumeExoplayer(); + } + }); + } + + // Called on Gecko's main thread. + @Override + public synchronized void play() { + runOnPlayerThread( + () -> { + if (mMediaDecoderPlayState == MediaDecoderPlayState.PLAY_STATE_PLAYING) { + return; + } + if (DEBUG) { + Log.d(LOGTAG, "MediaDecoder played."); + } + mMediaDecoderPlayState = MediaDecoderPlayState.PLAY_STATE_PLAYING; + resumeExoplayer(); + }); + } + + // Called on Gecko's main thread. + @Override + public synchronized void pause() { + runOnPlayerThread( + () -> { + if (mMediaDecoderPlayState != MediaDecoderPlayState.PLAY_STATE_PLAYING) { + return; + } + if (DEBUG) { + Log.d(LOGTAG, "MediaDecoder paused."); + } + mMediaDecoderPlayState = MediaDecoderPlayState.PLAY_STATE_PAUSED; + suspendExoplayer(); + }); + } + + private void suspendExoplayer() { + assertTrue(isPlayerThread()); + + if (mPlayer == null) { + return; + } + mExoplayerSuspended = true; + if (DEBUG) { + Log.d(LOGTAG, "suspend Exoplayer"); + } + mPlayer.setPlayWhenReady(false); + } + + private void resumeExoplayer() { + assertTrue(isPlayerThread()); + + if (mPlayer == null) { + return; + } + mExoplayerSuspended = false; + if (DEBUG) { + Log.d(LOGTAG, "resume Exoplayer"); + } + mPlayer.setPlayWhenReady(true); + } + + // Called on Gecko's main thread, when HLSDemuxer or HLSResource destructs. + @Override + public void release() { + if (DEBUG) { + Log.d(LOGTAG, "releasing ... id : " + mPlayerId); + } + + synchronized (this) { + if (mReleasing) { + return; + } else { + mReleasing = true; + } + } + + runOnPlayerThread( + () -> { + if (mPlayer != null) { + mPlayer.removeListener(this); + mPlayer.stop(); + mPlayer.release(); + mVRenderer = null; + mARenderer = null; + mPlayer = null; + } + if (mThread != null) { + mThread.quit(); + mThread = null; + } + mDemuxerCallbacks = null; + mResourceCallbacks = null; + mIsPlayerInitDone = false; + mIsDemuxerInitDone = false; + }); + } + + private void runOnPlayerThread(final Runnable task) { + assertTrue(mMainHandler != null); + if (isPlayerThread()) { + task.run(); + } else { + mMainHandler.post(task); + } + } + + private boolean isPlayerThread() { + return Thread.currentThread() == mMainHandler.getLooper().getThread(); + } + + private <T> T awaitPlayerThread(final Callable<T> task) { + assertTrue(!isPlayerThread()); + + try { + final FutureTask<T> wait = new FutureTask<T>(task); + mMainHandler.post(wait); + return wait.get(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsRendererBase.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsRendererBase.java new file mode 100644 index 0000000000..ecb7b93d61 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsRendererBase.java @@ -0,0 +1,340 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.util.Log; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.mozilla.geckoview.BuildConfig; +import org.mozilla.thirdparty.com.google.android.exoplayer2.BaseRenderer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; +import org.mozilla.thirdparty.com.google.android.exoplayer2.FormatHolder; +import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; +import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; + +public abstract class GeckoHlsRendererBase extends BaseRenderer { + protected static final int QUEUED_INPUT_SAMPLE_DURATION_THRESHOLD = 1000000; // 1sec + protected final FormatHolder mFormatHolder = new FormatHolder(); + /* + * DEBUG/LOGTAG will be set in the 2 subclass GeckoHlsAudioRenderer and + * GeckoHlsVideoRenderer, and we still wants to log message in the base class + * GeckoHlsRendererBase, so neither 'static' nor 'final' are applied to them. + */ + protected boolean DEBUG; + protected String LOGTAG; + // Notify GeckoHlsPlayer about renderer's status, i.e. data has arrived. + protected GeckoHlsPlayer.ComponentEventDispatcher mPlayerEventDispatcher; + + protected ConcurrentLinkedQueue<GeckoHLSSample> mDemuxedInputSamples = + new ConcurrentLinkedQueue<>(); + + protected ByteBuffer mInputBuffer = null; + protected ArrayList<Format> mFormats = new ArrayList<Format>(); + protected boolean mInitialized = false; + protected boolean mWaitingForData = true; + protected boolean mInputStreamEnded = false; + protected long mFirstSampleStartTime = Long.MIN_VALUE; + + protected abstract void createInputBuffer() throws ExoPlaybackException; + + protected abstract void handleReconfiguration(DecoderInputBuffer bufferForRead); + + protected abstract void handleFormatRead(DecoderInputBuffer bufferForRead) + throws ExoPlaybackException; + + protected abstract void handleEndOfStream(DecoderInputBuffer bufferForRead); + + protected abstract void handleSamplePreparation(DecoderInputBuffer bufferForRead); + + protected abstract void resetRenderer(); + + protected abstract boolean clearInputSamplesQueue(); + + protected abstract void notifyPlayerInputFormatChanged(Format newFormat); + + private DecoderInputBuffer mBufferForRead = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + private final DecoderInputBuffer mFlagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance(); + + protected void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + public GeckoHlsRendererBase( + final int trackType, final GeckoHlsPlayer.ComponentEventDispatcher eventDispatcher) { + super(trackType); + mPlayerEventDispatcher = eventDispatcher; + } + + private boolean isQueuedEnoughData() { + if (mDemuxedInputSamples.isEmpty()) { + return false; + } + + final Iterator<GeckoHLSSample> iter = mDemuxedInputSamples.iterator(); + long firstPTS = 0; + if (iter.hasNext()) { + final GeckoHLSSample sample = iter.next(); + firstPTS = sample.info.presentationTimeUs; + } + long lastPTS = firstPTS; + while (iter.hasNext()) { + final GeckoHLSSample sample = iter.next(); + lastPTS = sample.info.presentationTimeUs; + } + return Math.abs(lastPTS - firstPTS) > QUEUED_INPUT_SAMPLE_DURATION_THRESHOLD; + } + + public Format getFormat(final int index) { + assertTrue(index >= 0); + final Format fmt = index < mFormats.size() ? mFormats.get(index) : null; + if (DEBUG) { + Log.d(LOGTAG, "getFormat : index = " + index + ", format : " + fmt); + } + return fmt; + } + + public synchronized long getFirstSamplePTS() { + return mFirstSampleStartTime; + } + + public synchronized ConcurrentLinkedQueue<GeckoHLSSample> getQueuedSamples(final int number) { + final ConcurrentLinkedQueue<GeckoHLSSample> samples = + new ConcurrentLinkedQueue<GeckoHLSSample>(); + + GeckoHLSSample sample = null; + final int queuedSize = mDemuxedInputSamples.size(); + for (int i = 0; i < queuedSize; i++) { + if (i >= number) { + break; + } + sample = mDemuxedInputSamples.poll(); + samples.offer(sample); + } + + sample = samples.isEmpty() ? null : samples.peek(); + if (sample == null) { + if (DEBUG) { + Log.d(LOGTAG, "getQueuedSamples isEmpty, mWaitingForData = true !"); + } + mWaitingForData = true; + } else if (mFirstSampleStartTime == Long.MIN_VALUE) { + mFirstSampleStartTime = sample.info.presentationTimeUs; + if (DEBUG) { + Log.d(LOGTAG, "mFirstSampleStartTime = " + mFirstSampleStartTime); + } + } + return samples; + } + + protected void handleDrmInitChanged(final Format oldFormat, final Format newFormat) { + final Object oldDrmInit = oldFormat == null ? null : oldFormat.drmInitData; + final Object newDrnInit = newFormat.drmInitData; + + // TODO: Notify MFR if the content is encrypted or not. + if (newDrnInit != oldDrmInit) { + if (newDrnInit != null) { + } else { + } + } + } + + protected boolean canReconfigure(final Format oldFormat, final Format newFormat) { + // Referring to ExoPlayer's MediaCodecBaseRenderer, the default is set + // to false. Only override it in video renderer subclass. + return false; + } + + protected void prepareReconfiguration() { + // Referring to ExoPlayer's MediaCodec related renderers, only video + // renderer handles this. + } + + protected void updateCSDInfo(final Format format) { + // do nothing. + } + + protected void onInputFormatChanged(final Format newFormat) throws ExoPlaybackException { + Format oldFormat; + try { + oldFormat = mFormats.get(mFormats.size() - 1); + } catch (final IndexOutOfBoundsException e) { + oldFormat = null; + } + if (DEBUG) { + Log.d(LOGTAG, "[onInputFormatChanged] old : " + oldFormat + " => new : " + newFormat); + } + mFormats.add(newFormat); + handleDrmInitChanged(oldFormat, newFormat); + + if (mInitialized && canReconfigure(oldFormat, newFormat)) { + prepareReconfiguration(); + } else { + resetRenderer(); + maybeInitRenderer(); + } + + updateCSDInfo(newFormat); + notifyPlayerInputFormatChanged(newFormat); + } + + protected void maybeInitRenderer() throws ExoPlaybackException { + if (mInitialized || mFormats.size() == 0) { + return; + } + if (DEBUG) { + Log.d(LOGTAG, "Initializing ... "); + } + try { + createInputBuffer(); + mInitialized = true; + } catch (final OutOfMemoryError e) { + throw ExoPlaybackException.createForRenderer( + new RuntimeException(e), + getIndex(), + mFormats.isEmpty() ? null : getFormat(mFormats.size() - 1), + RendererCapabilities.FORMAT_HANDLED); + } + } + + /* + * The place we get demuxed data from HlsMediaSource(ExoPlayer). + * The data will then be converted to GeckoHLSSample and deliver to + * GeckoHlsDemuxerWrapper for further use. + * If the return value is ture, that means a GeckoHLSSample is queued + * successfully. We can try to feed more samples into queue. + * If the return value is false, that means we might encounter following + * situation 1) not initialized 2) input stream is ended 3) queue is full. + * 4) format changed. 5) exception happened. + */ + protected synchronized boolean feedInputBuffersQueue() throws ExoPlaybackException { + if (!mInitialized || mInputStreamEnded || isQueuedEnoughData()) { + // Need to reinitialize the renderer or the input stream has ended + // or we just reached the maximum queue size. + return false; + } + + mBufferForRead.data = mInputBuffer; + if (mBufferForRead.data != null) { + mBufferForRead.clear(); + } + + handleReconfiguration(mBufferForRead); + + // Read data from HlsMediaSource + int result = C.RESULT_NOTHING_READ; + try { + result = readSource(mFormatHolder, mBufferForRead, false); + } catch (final Exception e) { + Log.e(LOGTAG, "[feedInput] Exception when readSource :", e); + return false; + } + + if (result == C.RESULT_NOTHING_READ) { + return false; + } + + if (result == C.RESULT_FORMAT_READ) { + handleFormatRead(mBufferForRead); + return true; + } + + // We've read a buffer. + if (mBufferForRead.isEndOfStream()) { + if (DEBUG) { + Log.d(LOGTAG, "Now we're at the End Of Stream."); + } + handleEndOfStream(mBufferForRead); + return false; + } + + mBufferForRead.flip(); + + handleSamplePreparation(mBufferForRead); + + maybeNotifyDataArrived(); + return true; + } + + private void maybeNotifyDataArrived() { + if (mWaitingForData && isQueuedEnoughData()) { + if (DEBUG) { + Log.d(LOGTAG, "onDataArrived"); + } + mPlayerEventDispatcher.onDataArrived(getTrackType()); + mWaitingForData = false; + } + } + + private void readFormat() throws ExoPlaybackException { + mFlagsOnlyBuffer.clear(); + final int result = readSource(mFormatHolder, mFlagsOnlyBuffer, true); + if (result == C.RESULT_FORMAT_READ) { + onInputFormatChanged(mFormatHolder.format); + } + } + + @Override + protected void onEnabled(final boolean joining) { + // Do nothing. + } + + @Override + protected void onDisabled() { + mFormats.clear(); + resetRenderer(); + } + + @Override + public boolean isReady() { + return mFormats.size() != 0; + } + + @Override + public boolean isEnded() { + return mInputStreamEnded; + } + + @Override + protected synchronized void onPositionReset(final long positionUs, final boolean joining) { + if (DEBUG) { + Log.d(LOGTAG, "onPositionReset : positionUs = " + positionUs); + } + mInputStreamEnded = false; + if (mInitialized) { + clearInputSamplesQueue(); + } + } + + /* + * This is called by ExoPlayerImplInternal.java. + * ExoPlayer checks the status of renderer, i.e. isReady() / isEnded(), and + * calls renderer.render by passing its wall clock time. + */ + @Override + public void render(final long positionUs, final long elapsedRealtimeUs) + throws ExoPlaybackException { + if (BuildConfig.DEBUG_BUILD) { + Log.d(LOGTAG, "positionUs = " + positionUs + ", mInputStreamEnded = " + mInputStreamEnded); + } + if (mInputStreamEnded) { + return; + } + if (mFormats.size() == 0) { + readFormat(); + } + + maybeInitRenderer(); + while (feedInputBuffersQueue()) { + // Do nothing + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsVideoRenderer.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsVideoRenderer.java new file mode 100644 index 0000000000..28f7bad5cf --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoHlsVideoRenderer.java @@ -0,0 +1,502 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import android.util.Log; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.mozilla.geckoview.BuildConfig; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; +import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; +import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecInfo; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import org.mozilla.thirdparty.com.google.android.exoplayer2.mediacodec.MediaCodecUtil; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.MimeTypes; + +public class GeckoHlsVideoRenderer extends GeckoHlsRendererBase { + /* + * By configuring these states, initialization data is provided for + * ExoPlayer's HlsMediaSource to parse HLS bitstream and then provide samples + * starting with an Access Unit Delimiter including SPS/PPS for TS, + * and provide samples starting with an AUD without SPS/PPS for FMP4. + */ + private enum RECONFIGURATION_STATE { + NONE, + WRITE_PENDING, + QUEUE_PENDING + } + + private boolean mRendererReconfigured; + private RECONFIGURATION_STATE mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; + + // A list of the formats which may be included in the bitstream. + private Format[] mStreamFormats; + // The max width/height/inputBufferSize for specific codec format. + private CodecMaxValues mCodecMaxValues; + // A temporary queue for samples whose duration is not calculated yet. + private ConcurrentLinkedQueue<GeckoHLSSample> mDemuxedNoDurationSamples = + new ConcurrentLinkedQueue<>(); + + // Contain CSD-0(SPS)/CSD-1(PPS) information (in AnnexB format) for + // prepending each keyframe. When video format changes, this information + // changes accordingly. + private byte[] mCSDInfo = null; + + public GeckoHlsVideoRenderer(final GeckoHlsPlayer.ComponentEventDispatcher eventDispatcher) { + super(C.TRACK_TYPE_VIDEO, eventDispatcher); + LOGTAG = getClass().getSimpleName(); + DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + } + + @Override + public final int supportsMixedMimeTypeAdaptation() { + return ADAPTIVE_NOT_SEAMLESS; + } + + @Override + public final int supportsFormat(final Format format) { + /* + * FORMAT_EXCEEDS_CAPABILITIES : The Renderer is capable of rendering + * formats with the same mime type, but + * the properties of the format exceed + * the renderer's capability. + * FORMAT_UNSUPPORTED_SUBTYPE : The Renderer is a general purpose + * renderer for formats of the same + * top-level type, but is not capable of + * rendering the format or any other format + * with the same mime type because the + * sub-type is not supported. + * FORMAT_UNSUPPORTED_TYPE : The Renderer is not capable of rendering + * the format, either because it does not support + * the format's top-level type, or because it's + * a specialized renderer for a different mime type. + * ADAPTIVE_NOT_SEAMLESS : The Renderer can adapt between formats, + * but may suffer a brief discontinuity (~50-100ms) + * when adaptation occurs. + * ADAPTIVE_SEAMLESS : The Renderer can seamlessly adapt between formats. + */ + final String mimeType = format.sampleMimeType; + if (!MimeTypes.isVideo(mimeType)) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + } + + List<MediaCodecInfo> decoderInfos = null; + try { + final MediaCodecSelector mediaCodecSelector = MediaCodecSelector.DEFAULT; + decoderInfos = mediaCodecSelector.getDecoderInfos(mimeType, false, false); + } catch (final MediaCodecUtil.DecoderQueryException e) { + Log.e(LOGTAG, e.getMessage()); + } + if (decoderInfos == null || decoderInfos.isEmpty()) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE); + } + + boolean decoderCapable = false; + MediaCodecInfo info = null; + for (final MediaCodecInfo i : decoderInfos) { + if (i.isCodecSupported(format)) { + decoderCapable = true; + info = i; + } + } + if (decoderCapable && format.width > 0 && format.height > 0) { + decoderCapable = + info.isVideoSizeAndRateSupportedV21(format.width, format.height, format.frameRate); + } + + return RendererCapabilities.create( + decoderCapable ? FORMAT_HANDLED : FORMAT_EXCEEDS_CAPABILITIES, + info != null && info.adaptive ? ADAPTIVE_SEAMLESS : ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED); + } + + @Override + protected final void createInputBuffer() throws ExoPlaybackException { + assertTrue(mFormats.size() > 0); + // Calculate maximum size which might be used for target format. + final Format currentFormat = mFormats.get(mFormats.size() - 1); + mCodecMaxValues = getCodecMaxValues(currentFormat, mStreamFormats); + // Create a buffer with maximal size for reading source. + // Note : Though we are able to dynamically enlarge buffer size by + // creating DecoderInputBuffer with specific BufferReplacementMode, we + // still allocate a calculated max size buffer for it at first to reduce + // runtime overhead. + try { + mInputBuffer = ByteBuffer.wrap(new byte[mCodecMaxValues.inputSize]); + } catch (final OutOfMemoryError e) { + Log.e(LOGTAG, "cannot allocate input buffer of size " + mCodecMaxValues.inputSize, e); + throw ExoPlaybackException.createForRenderer( + new Exception(e), + getIndex(), + mFormats.isEmpty() ? null : getFormat(mFormats.size() - 1), + RendererCapabilities.FORMAT_HANDLED); + } + } + + @Override + protected void resetRenderer() { + if (DEBUG) { + Log.d(LOGTAG, "[resetRenderer] mInitialized = " + mInitialized); + } + if (mInitialized) { + mRendererReconfigured = false; + mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; + mInputBuffer = null; + mCSDInfo = null; + mInitialized = false; + } + } + + @Override + protected void handleReconfiguration(final DecoderInputBuffer bufferForRead) { + // For adaptive reconfiguration OMX decoders expect all reconfiguration + // data to be supplied at the start of the buffer that also contains + // the first frame in the new format. + assertTrue(mFormats.size() > 0); + if (mRendererReconfigurationState == RECONFIGURATION_STATE.WRITE_PENDING) { + if (bufferForRead.data == null) { + if (DEBUG) { + Log.d(LOGTAG, "[feedInput][WRITE_PENDING] bufferForRead.data is not initialized."); + } + return; + } + if (DEBUG) { + Log.d(LOGTAG, "[feedInput][WRITE_PENDING] put initialization data"); + } + final Format currentFormat = mFormats.get(mFormats.size() - 1); + for (int i = 0; i < currentFormat.initializationData.size(); i++) { + final byte[] data = currentFormat.initializationData.get(i); + bufferForRead.data.put(data); + } + mRendererReconfigurationState = RECONFIGURATION_STATE.QUEUE_PENDING; + } + } + + @Override + protected void handleFormatRead(final DecoderInputBuffer bufferForRead) + throws ExoPlaybackException { + if (mRendererReconfigurationState == RECONFIGURATION_STATE.QUEUE_PENDING) { + if (DEBUG) { + Log.d(LOGTAG, "[feedInput][QUEUE_PENDING] 2 formats in a row."); + } + // We received two formats in a row. Clear the current buffer of any reconfiguration data + // associated with the first format. + bufferForRead.clear(); + mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; + } + onInputFormatChanged(mFormatHolder.format); + } + + @Override + protected void handleEndOfStream(final DecoderInputBuffer bufferForRead) { + if (mRendererReconfigurationState == RECONFIGURATION_STATE.QUEUE_PENDING) { + if (DEBUG) { + Log.d(LOGTAG, "[feedInput][QUEUE_PENDING] isEndOfStream."); + } + // We received a new format immediately before the end of the stream. We need to clear + // the corresponding reconfiguration data from the current buffer, but re-write it into + // a subsequent buffer if there are any (e.g. if the user seeks backwards). + bufferForRead.clear(); + mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; + } + mInputStreamEnded = true; + final GeckoHLSSample sample = GeckoHLSSample.EOS; + calculatDuration(sample); + } + + @Override + protected void handleSamplePreparation(final DecoderInputBuffer bufferForRead) { + final int csdInfoSize = mCSDInfo != null ? mCSDInfo.length : 0; + final int dataSize = bufferForRead.data.limit(); + final int size = bufferForRead.isKeyFrame() ? csdInfoSize + dataSize : dataSize; + final byte[] realData = new byte[size]; + if (bufferForRead.isKeyFrame()) { + // Prepend the CSD information to the sample if it's a key frame. + System.arraycopy(mCSDInfo, 0, realData, 0, csdInfoSize); + bufferForRead.data.get(realData, csdInfoSize, dataSize); + } else { + bufferForRead.data.get(realData, 0, dataSize); + } + final ByteBuffer buffer = ByteBuffer.wrap(realData); + mInputBuffer = bufferForRead.data; + mInputBuffer.clear(); + + final CryptoInfo cryptoInfo = + bufferForRead.isEncrypted() ? bufferForRead.cryptoInfo.getFrameworkCryptoInfoV16() : null; + final BufferInfo bufferInfo = new BufferInfo(); + // Flags in DecoderInputBuffer are synced with MediaCodec Buffer flags. + int flags = 0; + flags |= bufferForRead.isKeyFrame() ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0; + flags |= bufferForRead.isEndOfStream() ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0; + bufferInfo.set(0, size, bufferForRead.timeUs, flags); + + assertTrue(mFormats.size() > 0); + // We add a new format in the list once format changes, so the formatIndex + // should indicate to the last(latest) format. + final GeckoHLSSample sample = + GeckoHLSSample.create(buffer, bufferInfo, cryptoInfo, mFormats.size() - 1); + + // There's no duration information from the ExoPlayer's sample, we need + // to calculate it. + calculatDuration(sample); + mRendererReconfigurationState = RECONFIGURATION_STATE.NONE; + } + + @Override + protected void onPositionReset(final long positionUs, final boolean joining) { + super.onPositionReset(positionUs, joining); + if (mInitialized && mRendererReconfigured && mFormats.size() != 0) { + if (DEBUG) { + Log.d(LOGTAG, "[onPositionReset] WRITE_PENDING"); + } + // Any reconfiguration data that we put shortly before the reset + // may be invalid. We avoid this issue by sending reconfiguration + // data following every position reset. + mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; + } + } + + @Override + protected boolean clearInputSamplesQueue() { + if (DEBUG) { + Log.d(LOGTAG, "clearInputSamplesQueue"); + } + mDemuxedInputSamples.clear(); + mDemuxedNoDurationSamples.clear(); + return true; + } + + @Override + protected boolean canReconfigure(final Format oldFormat, final Format newFormat) { + final boolean canReconfig = + areAdaptationCompatible(oldFormat, newFormat) + && newFormat.width <= mCodecMaxValues.width + && newFormat.height <= mCodecMaxValues.height + && newFormat.maxInputSize <= mCodecMaxValues.inputSize; + if (DEBUG) { + Log.d(LOGTAG, "[canReconfigure] : " + canReconfig); + } + return canReconfig; + } + + @Override + protected void prepareReconfiguration() { + if (DEBUG) { + Log.d(LOGTAG, "[onInputFormatChanged] starting reconfiguration !"); + } + mRendererReconfigured = true; + mRendererReconfigurationState = RECONFIGURATION_STATE.WRITE_PENDING; + } + + @Override + protected void updateCSDInfo(final Format format) { + int size = 0; + for (int i = 0; i < format.initializationData.size(); i++) { + size += format.initializationData.get(i).length; + } + int startPos = 0; + mCSDInfo = new byte[size]; + for (int i = 0; i < format.initializationData.size(); i++) { + final byte[] data = format.initializationData.get(i); + System.arraycopy(data, 0, mCSDInfo, startPos, data.length); + startPos += data.length; + } + if (DEBUG) { + Log.d(LOGTAG, "mCSDInfo [" + Utils.bytesToHex(mCSDInfo) + "]"); + } + } + + @Override + protected void notifyPlayerInputFormatChanged(final Format newFormat) { + mPlayerEventDispatcher.onVideoInputFormatChanged(newFormat); + } + + private void calculateSamplesWithin(final GeckoHLSSample[] samples, final int range) { + // Calculate the first 'range' elements. + for (int i = 0; i < range; i++) { + // Comparing among samples in the window. + for (int j = -2; j < 14; j++) { + if (i + j >= 0 + && i + j < range + && samples[i + j].info.presentationTimeUs > samples[i].info.presentationTimeUs) { + samples[i].duration = + Math.min( + samples[i].duration, + samples[i + j].info.presentationTimeUs - samples[i].info.presentationTimeUs); + } + } + } + } + + private void calculatDuration(final GeckoHLSSample inputSample) { + /* + * NOTE : + * Since we customized renderer as a demuxer. Here we're not able to + * obtain duration from the DecoderInputBuffer as there's no duration inside. + * So we calcualte it by referring to nearby samples' timestamp. + * A temporary queue |mDemuxedNoDurationSamples| is used to queue demuxed + * samples from HlsMediaSource which have no duration information at first. + * We're choosing 16 as the comparing window size, because it's commonly + * used as a GOP size. + * Considering there're 16 demuxed samples in the _no duration_ queue already, + * e.g. |-2|-1|0|1|2|3|4|5|6|...|13| + * Once a new demuxed(No duration) sample X (17th) is put into the + * temporary queue, + * e.g. |-2|-1|0|1|2|3|4|5|6|...|13|X| + * we are able to calculate the correct duration for sample 0 by finding + * the closest but greater pts than sample 0 among these 16 samples, + * here, let's say sample -2 to 13. + */ + if (inputSample != null) { + mDemuxedNoDurationSamples.offer(inputSample); + } + final int sizeOfNoDura = mDemuxedNoDurationSamples.size(); + // A calculation window we've ever found suitable for both HLS TS & FMP4. + final int range = sizeOfNoDura >= 17 ? 17 : sizeOfNoDura; + final GeckoHLSSample[] inputArray = + mDemuxedNoDurationSamples.toArray(new GeckoHLSSample[sizeOfNoDura]); + if (range >= 17 && !mInputStreamEnded) { + calculateSamplesWithin(inputArray, range); + + final GeckoHLSSample toQueue = mDemuxedNoDurationSamples.poll(); + mDemuxedInputSamples.offer(toQueue); + if (BuildConfig.DEBUG_BUILD) { + Log.d( + LOGTAG, + "Demuxed sample PTS : " + + toQueue.info.presentationTimeUs + + ", duration :" + + toQueue.duration + + ", isKeyFrame(" + + toQueue.isKeyFrame() + + ", formatIndex(" + + toQueue.formatIndex + + "), queue size : " + + mDemuxedInputSamples.size() + + ", NoDuQueue size : " + + mDemuxedNoDurationSamples.size()); + } + } else if (mInputStreamEnded) { + calculateSamplesWithin(inputArray, sizeOfNoDura); + + // NOTE : We're not able to calculate the duration for the last sample. + // A workaround here is to assign a close duration to it. + long prevDuration = 33333; + GeckoHLSSample sample = null; + for (sample = mDemuxedNoDurationSamples.poll(); + sample != null; + sample = mDemuxedNoDurationSamples.poll()) { + if (sample.duration == Long.MAX_VALUE) { + sample.duration = prevDuration; + if (DEBUG) { + Log.d(LOGTAG, "Adjust the PTS of the last sample to " + sample.duration + " (us)"); + } + } + prevDuration = sample.duration; + if (DEBUG) { + Log.d( + LOGTAG, + "last loop to offer samples - PTS : " + + sample.info.presentationTimeUs + + ", Duration : " + + sample.duration + + ", isEOS : " + + sample.isEOS()); + } + mDemuxedInputSamples.offer(sample); + } + } + } + + // Return the time of first keyframe sample in the queue. + // If there's no key frame in the queue, return the MAX_VALUE so + // MFR won't mistake for that which the decode is getting slow. + public long getNextKeyFrameTime() { + long nextKeyFrameTime = Long.MAX_VALUE; + for (final GeckoHLSSample sample : mDemuxedInputSamples) { + if (sample != null && (sample.info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { + nextKeyFrameTime = sample.info.presentationTimeUs; + break; + } + } + return nextKeyFrameTime; + } + + @Override + protected void onStreamChanged(final Format[] formats, final long offsetUs) { + mStreamFormats = formats; + } + + private static CodecMaxValues getCodecMaxValues( + final Format format, final Format[] streamFormats) { + int maxWidth = format.width; + int maxHeight = format.height; + int maxInputSize = getMaxInputSize(format); + for (final Format streamFormat : streamFormats) { + if (areAdaptationCompatible(format, streamFormat)) { + maxWidth = Math.max(maxWidth, streamFormat.width); + maxHeight = Math.max(maxHeight, streamFormat.height); + maxInputSize = Math.max(maxInputSize, getMaxInputSize(streamFormat)); + } + } + return new CodecMaxValues(maxWidth, maxHeight, maxInputSize); + } + + private static int getMaxInputSize(final Format format) { + if (format.maxInputSize != Format.NO_VALUE) { + // The format defines an explicit maximum input size. + return format.maxInputSize; + } + + if (format.width == Format.NO_VALUE || format.height == Format.NO_VALUE) { + // We can't infer a maximum input size without video dimensions. + return Format.NO_VALUE; + } + + // Attempt to infer a maximum input size from the format. + final int maxPixels; + final int minCompressionRatio; + switch (format.sampleMimeType) { + case MimeTypes.VIDEO_H264: + // Round up width/height to an integer number of macroblocks. + maxPixels = ((format.width + 15) / 16) * ((format.height + 15) / 16) * 16 * 16; + minCompressionRatio = 2; + break; + default: + // Leave the default max input size. + return Format.NO_VALUE; + } + // Estimate the maximum input size assuming three channel 4:2:0 subsampled input frames. + return (maxPixels * 3) / (2 * minCompressionRatio); + } + + private static boolean areAdaptationCompatible(final Format first, final Format second) { + return first.sampleMimeType.equals(second.sampleMimeType) + && getRotationDegrees(first) == getRotationDegrees(second); + } + + private static int getRotationDegrees(final Format format) { + return format.rotationDegrees == Format.NO_VALUE ? 0 : format.rotationDegrees; + } + + private static final class CodecMaxValues { + public final int width; + public final int height; + public final int inputSize; + + public CodecMaxValues(final int width, final int height, final int inputSize) { + this.width = width; + this.height = height; + this.inputSize = inputSize; + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrm.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrm.java new file mode 100644 index 0000000000..75dc7b2a80 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrm.java @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCrypto; + +public interface GeckoMediaDrm { + interface Callbacks { + void onSessionCreated(int createSessionToken, int promiseId, byte[] sessionId, byte[] request); + + void onSessionUpdated(int promiseId, byte[] sessionId); + + void onSessionClosed(int promiseId, byte[] sessionId); + + void onSessionMessage(byte[] sessionId, int sessionMessageType, byte[] request); + + void onSessionError(byte[] sessionId, String message); + + void onSessionBatchedKeyChanged(byte[] sessionId, SessionKeyInfo[] keyInfos); + + // All failure cases should go through this function. + void onRejectPromise(int promiseId, String message); + } + + void setCallbacks(Callbacks callbacks); + + void createSession(int createSessionToken, int promiseId, String initDataType, byte[] initData); + + void updateSession(int promiseId, String sessionId, byte[] response); + + void closeSession(int promiseId, String sessionId); + + void release(); + + MediaCrypto getMediaCrypto(); + + void setServerCertificate(final byte[] cert); +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV21.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV21.java new file mode 100644 index 0000000000..9d098a303f --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV21.java @@ -0,0 +1,766 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.annotation.SuppressLint; +import android.media.DeniedByServerException; +import android.media.MediaCrypto; +import android.media.MediaDrm; +import android.media.NotProvisionedException; +import android.os.AsyncTask; +import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; +import android.util.Log; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.UUID; +import org.mozilla.gecko.util.ProxySelector; + +public class GeckoMediaDrmBridgeV21 implements GeckoMediaDrm { + protected final String LOGTAG; + private static final String INVALID_SESSION_ID = "Invalid"; + private static final String WIDEVINE_KEY_SYSTEM = "com.widevine.alpha"; + private static final boolean DEBUG = false; + private static final UUID WIDEVINE_SCHEME_UUID = + new UUID(0xedef8ba979d64aceL, 0xa3c827dcd51d21edL); + private static final int MAX_PROMISE_ID = Integer.MAX_VALUE; + // MediaDrm.KeyStatus information listener is supported on M+, adding a + // dummy key id to report key status. + private static final byte[] DUMMY_KEY_ID = new byte[] {0}; + + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + private UUID mSchemeUUID; + private Handler mHandler; + PostRequestTask mProvisionTask; + private HandlerThread mHandlerThread; + private ByteBuffer mCryptoSessionId; + + // mProvisioningPromiseId is great than 0 only during provisioning. + private int mProvisioningPromiseId; + private HashSet<ByteBuffer> mSessionIds; + private HashMap<ByteBuffer, String> mSessionMIMETypes; + private ArrayDeque<PendingCreateSessionData> mPendingCreateSessionDataQueue; + private PendingKeyRequest mPendingKeyRequest; + private GeckoMediaDrm.Callbacks mCallbacks; + + private MediaCrypto mCrypto; + protected MediaDrm mDrm; + + public static final int LICENSE_REQUEST_INITIAL = 0; /*MediaKeyMessageType::License_request*/ + public static final int LICENSE_REQUEST_RENEWAL = 1; /*MediaKeyMessageType::License_renewal*/ + public static final int LICENSE_REQUEST_RELEASE = 2; /*MediaKeyMessageType::License_release*/ + + // Store session data while provisioning + private static class PendingCreateSessionData { + public final int mToken; + public final int mPromiseId; + public final byte[] mInitData; + public final String mMimeType; + + private PendingCreateSessionData( + final int token, final int promiseId, final byte[] initData, final String mimeType) { + mToken = token; + mPromiseId = promiseId; + mInitData = initData; + mMimeType = mimeType; + } + } + + private static class PendingKeyRequest { + public final ByteBuffer mSession; + public final byte[] mData; + public final String mMimeType; + + private PendingKeyRequest(final ByteBuffer session, final byte[] data, final String mimeType) { + mSession = session; + mData = data; + mMimeType = mimeType; + } + } + + public boolean isSecureDecoderComonentRequired(final String mimeType) { + if (mCrypto != null) { + return mCrypto.requiresSecureDecoderComponent(mimeType); + } + return false; + } + + private static void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + @SuppressLint("WrongConstant") + private void configureVendorSpecificProperty() { + assertTrue(mDrm != null); + if (mDrm == null) { + return; + } + // Support L3 for now + mDrm.setPropertyString("securityLevel", "L3"); + // Refer to chromium, set multi-session mode for Widevine. + if (mSchemeUUID.equals(WIDEVINE_SCHEME_UUID)) { + mDrm.setPropertyString("privacyMode", "enable"); + mDrm.setPropertyString("sessionSharing", "enable"); + } + } + + GeckoMediaDrmBridgeV21(final String keySystem) throws Exception { + LOGTAG = getClass().getSimpleName(); + if (DEBUG) Log.d(LOGTAG, "GeckoMediaDrmBridgeV21 ctor"); + + mProvisioningPromiseId = 0; + mSessionIds = new HashSet<ByteBuffer>(); + mSessionMIMETypes = new HashMap<ByteBuffer, String>(); + mPendingCreateSessionDataQueue = new ArrayDeque<PendingCreateSessionData>(); + + mSchemeUUID = convertKeySystemToSchemeUUID(keySystem); + mCryptoSessionId = null; + + if (DEBUG) Log.d(LOGTAG, "mSchemeUUID : " + mSchemeUUID.toString()); + + // The caller of GeckoMediaDrmBridgeV21 ctor should handle exceptions + // threw by the following steps. + mDrm = new MediaDrm(mSchemeUUID); + configureVendorSpecificProperty(); + mDrm.setOnEventListener(new MediaDrmListener()); + try { + // ensureMediaCryptoCreated may cause NotProvisionedException for the first time use. + // Need to start provisioning with a dummy promise id. + ensureMediaCryptoCreated(); + } catch (final android.media.NotProvisionedException e) { + if (DEBUG) Log.d(LOGTAG, "Device not provisioned:" + e.getMessage()); + startProvisioning(MAX_PROMISE_ID); + } + } + + @Override + public void setCallbacks(final GeckoMediaDrm.Callbacks callbacks) { + assertTrue(callbacks != null); + mCallbacks = callbacks; + } + + @Override + public void createSession( + final int createSessionToken, + final int promiseId, + final String initDataType, + final byte[] initData) { + if (DEBUG) Log.d(LOGTAG, "createSession()"); + if (mDrm == null) { + onRejectPromise(promiseId, "MediaDrm instance doesn't exist !!"); + return; + } + + if (mProvisioningPromiseId > 0 && mCrypto == null) { + if (DEBUG) Log.d(LOGTAG, "Pending createSession because it's provisioning !"); + savePendingCreateSessionData( + createSessionToken, promiseId, + initData, initDataType); + return; + } + + ByteBuffer sessionId = null; + try { + final boolean hasMediaCrypto = ensureMediaCryptoCreated(); + if (!hasMediaCrypto) { + onRejectPromise(promiseId, "MediaCrypto intance is not created !"); + return; + } + + sessionId = openSession(); + if (sessionId == null) { + onRejectPromise(promiseId, "Cannot get a session id from MediaDrm !"); + return; + } + + final MediaDrm.KeyRequest request = getKeyRequest(sessionId, initData, initDataType); + if (request == null) { + mDrm.closeSession(sessionId.array()); + onRejectPromise(promiseId, "Cannot get a key request from MediaDrm !"); + return; + } + onSessionCreated(createSessionToken, promiseId, sessionId.array(), request.getData()); + onSessionMessage(sessionId.array(), LICENSE_REQUEST_INITIAL, request.getData()); + mSessionMIMETypes.put(sessionId, initDataType); + mSessionIds.add(sessionId); + if (DEBUG) + Log.d( + LOGTAG, + " StringID : " + new String(sessionId.array(), UTF_8) + " is put into mSessionIds "); + } catch (final android.media.NotProvisionedException e) { + if (DEBUG) Log.d(LOGTAG, "Device not provisioned:" + e.getMessage()); + if (sessionId != null) { + // The promise of this createSession will be either resolved + // or rejected after provisioning. + mDrm.closeSession(sessionId.array()); + } + savePendingCreateSessionData( + createSessionToken, promiseId, + initData, initDataType); + startProvisioning(promiseId); + } + } + + @Override + public void updateSession(final int promiseId, final String sessionId, final byte[] response) { + if (DEBUG) Log.d(LOGTAG, "updateSession(), sessionId = " + sessionId); + if (mDrm == null) { + onRejectPromise(promiseId, "MediaDrm instance doesn't exist !!"); + return; + } + + final ByteBuffer session = ByteBuffer.wrap(sessionId.getBytes(UTF_8)); + if (!sessionExists(session)) { + onRejectPromise(promiseId, "Invalid session during updateSession."); + return; + } + + try { + final byte[] keySetId = mDrm.provideKeyResponse(session.array(), response); + if (DEBUG) { + final HashMap<String, String> infoMap = mDrm.queryKeyStatus(session.array()); + for (final String strKey : infoMap.keySet()) { + final String strValue = infoMap.get(strKey); + Log.d(LOGTAG, "InfoMap : key(" + strKey + ")/value(" + strValue + ")"); + } + } + HandleKeyStatusChangeByDummyKey(sessionId); + onSessionUpdated(promiseId, session.array()); + return; + } catch (final NotProvisionedException | DeniedByServerException | IllegalStateException e) { + if (DEBUG) Log.d(LOGTAG, "Failed to provide key response:", e); + onSessionError(session.array(), "Got exception during updateSession."); + onRejectPromise(promiseId, "Got exception during updateSession."); + } + release(); + return; + } + + @Override + public void closeSession(final int promiseId, final String sessionId) { + if (DEBUG) Log.d(LOGTAG, "closeSession()"); + if (mDrm == null) { + onRejectPromise(promiseId, "MediaDrm instance doesn't exist !!"); + return; + } + + final ByteBuffer session = ByteBuffer.wrap(sessionId.getBytes(UTF_8)); + mSessionIds.remove(session); + mDrm.closeSession(session.array()); + onSessionClosed(promiseId, session.array()); + } + + @Override + public void release() { + if (DEBUG) Log.d(LOGTAG, "release()"); + if (mProvisionTask != null) { + mProvisionTask.cancel(true); + mProvisionTask = null; + } + if (mProvisioningPromiseId > 0) { + onRejectPromise(mProvisioningPromiseId, "Releasing ... reject provisioning session."); + mProvisioningPromiseId = 0; + } + if (mPendingKeyRequest != null) { + mPendingKeyRequest = null; + } + while (!mPendingCreateSessionDataQueue.isEmpty()) { + final PendingCreateSessionData pendingData = mPendingCreateSessionDataQueue.poll(); + if (pendingData != null) { + onRejectPromise(pendingData.mPromiseId, "Releasing ... reject all pending sessions."); + } + } + mPendingCreateSessionDataQueue = null; + + if (mDrm != null) { + for (final ByteBuffer session : mSessionIds) { + mDrm.closeSession(session.array()); + } + mDrm.release(); + mDrm = null; + } + mSessionIds.clear(); + mSessionIds = null; + mSessionMIMETypes.clear(); + mSessionMIMETypes = null; + + mCryptoSessionId = null; + if (mCrypto != null) { + mCrypto.release(); + mCrypto = null; + } + if (mHandlerThread != null) { + mHandlerThread.quitSafely(); + mHandlerThread = null; + } + mHandler = null; + } + + @Override + public MediaCrypto getMediaCrypto() { + if (DEBUG) Log.d(LOGTAG, "getMediaCrypto()"); + return mCrypto; + } + + @SuppressLint("WrongConstant") + @Override + public void setServerCertificate(final byte[] cert) { + if (DEBUG) Log.d(LOGTAG, "setServerCertificate()"); + if (mDrm == null) { + throw new IllegalStateException("MediaDrm instance doesn't exist !!"); + } + mDrm.setPropertyByteArray("serviceCertificate", cert); + return; + } + + protected void HandleKeyStatusChangeByDummyKey(final String sessionId) { + final SessionKeyInfo[] keyInfos = new SessionKeyInfo[1]; + keyInfos[0] = new SessionKeyInfo(DUMMY_KEY_ID, MediaDrm.KeyStatus.STATUS_USABLE); + onSessionBatchedKeyChanged(sessionId.getBytes(), keyInfos); + if (DEBUG) Log.d(LOGTAG, "Key successfully added for session " + sessionId); + } + + protected void onSessionCreated( + final int createSessionToken, + final int promiseId, + final byte[] sessionId, + final byte[] request) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionCreated(createSessionToken, promiseId, sessionId, request); + } + } + + protected void onSessionUpdated(final int promiseId, final byte[] sessionId) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionUpdated(promiseId, sessionId); + } + } + + protected void onSessionClosed(final int promiseId, final byte[] sessionId) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionClosed(promiseId, sessionId); + } + } + + protected void onSessionMessage( + final byte[] sessionId, final int sessionMessageType, final byte[] request) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionMessage(sessionId, sessionMessageType, request); + } + } + + protected void onSessionError(final byte[] sessionId, final String message) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionError(sessionId, message); + } + } + + protected void onSessionBatchedKeyChanged( + final byte[] sessionId, final SessionKeyInfo[] keyInfos) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onSessionBatchedKeyChanged(sessionId, keyInfos); + } + } + + protected void onRejectPromise(final int promiseId, final String message) { + assertTrue(mCallbacks != null); + if (mCallbacks != null) { + mCallbacks.onRejectPromise(promiseId, message); + } + } + + private MediaDrm.KeyRequest getKeyRequest( + final ByteBuffer aSession, final byte[] data, final String mimeType) + throws android.media.NotProvisionedException { + if (mProvisioningPromiseId > 0) { + if (DEBUG) Log.d(LOGTAG, "Now provisioning"); + return null; + } + + try { + final HashMap<String, String> optionalParameters = new HashMap<String, String>(); + return mDrm.getKeyRequest( + aSession.array(), data, mimeType, MediaDrm.KEY_TYPE_STREAMING, optionalParameters); + } catch (final Exception e) { + Log.e(LOGTAG, "Got excpetion during MediaDrm.getKeyRequest", e); + } + return null; + } + + private class MediaDrmListener implements MediaDrm.OnEventListener { + @Override + public void onEvent( + final MediaDrm mediaDrm, + final byte[] sessionArray, + final int event, + final int extra, + final byte[] data) { + if (DEBUG) Log.d(LOGTAG, "MediaDrmListener.onEvent()"); + if (sessionArray == null) { + if (DEBUG) Log.d(LOGTAG, "MediaDrmListener: Null session."); + return; + } + final ByteBuffer session = ByteBuffer.wrap(sessionArray); + if (!sessionExists(session)) { + if (DEBUG) Log.d(LOGTAG, "MediaDrmListener: Invalid session."); + return; + } + // On L, these events are treated as exceptions and handled correspondingly. + // Leaving this code block for logging message. + switch (event) { + case MediaDrm.EVENT_PROVISION_REQUIRED: + if (DEBUG) Log.d(LOGTAG, "MediaDrm.EVENT_PROVISION_REQUIRED"); + break; + case MediaDrm.EVENT_KEY_REQUIRED: + if (DEBUG) + Log.d( + LOGTAG, + "MediaDrm.EVENT_KEY_REQUIRED, sessionId=" + new String(session.array(), UTF_8)); + final String mimeType = mSessionMIMETypes.get(session); + MediaDrm.KeyRequest request = null; + try { + request = getKeyRequest(session, data, mimeType); + } catch (final android.media.NotProvisionedException e) { + Log.w(LOGTAG, "MediaDrm.EVENT_KEY_REQUIRED, Device not provisioned.", e); + startProvisioning(MAX_PROMISE_ID); + mPendingKeyRequest = new PendingKeyRequest(session, data, mimeType); + return; + } + requestLicense(sessionArray, request); + break; + case MediaDrm.EVENT_KEY_EXPIRED: + if (DEBUG) + Log.d( + LOGTAG, + "MediaDrm.EVENT_KEY_EXPIRED, sessionId=" + new String(session.array(), UTF_8)); + break; + case MediaDrm.EVENT_VENDOR_DEFINED: + if (DEBUG) + Log.d( + LOGTAG, + "MediaDrm.EVENT_VENDOR_DEFINED, sessionId=" + new String(session.array(), UTF_8)); + break; + case MediaDrm.EVENT_SESSION_RECLAIMED: + if (DEBUG) + Log.d( + LOGTAG, + "MediaDrm.EVENT_SESSION_RECLAIMED, sessionId=" + + new String(session.array(), UTF_8)); + break; + default: + if (DEBUG) Log.d(LOGTAG, "Invalid DRM event " + event); + return; + } + } + } + + private ByteBuffer openSession() throws android.media.NotProvisionedException { + try { + final byte[] sessionId = mDrm.openSession(); + // ByteBuffer.wrap() is backed by the byte[]. Make a clone here in + // case the underlying byte[] is modified. + return ByteBuffer.wrap(sessionId.clone()); + } catch (final android.media.NotProvisionedException e) { + // Throw NotProvisionedException so that we can startProvisioning(). + throw e; + } catch (final java.lang.RuntimeException e) { + if (DEBUG) Log.d(LOGTAG, "Cannot open a new session:" + e.getMessage()); + release(); + return null; + } catch (final android.media.MediaDrmException e) { + // Other MediaDrmExceptions (e.g. ResourceBusyException) are not + // recoverable. + release(); + return null; + } + } + + protected boolean sessionExists(final ByteBuffer session) { + if (mCryptoSessionId == null) { + if (DEBUG) + Log.d(LOGTAG, "Session doesn't exist because media crypto session is not created."); + return false; + } + if (session == null) { + if (DEBUG) Log.d(LOGTAG, "Session is null, not in map !"); + return false; + } + return !session.equals(mCryptoSessionId) && mSessionIds.contains(session); + } + + private class PostRequestTask extends AsyncTask<Void, Void, Void> { + private static final String LOGTAG = "PostRequestTask"; + + private int mPromiseId; + private String mURL; + private byte[] mDrmRequest; + private byte[] mResponseBody; + + PostRequestTask(final int promiseId, final String url, final byte[] drmRequest) { + this.mPromiseId = promiseId; + this.mURL = url; + this.mDrmRequest = drmRequest; + } + + @Override + protected Void doInBackground(final Void... params) { + HttpURLConnection urlConnection = null; + BufferedReader in = null; + try { + final URI finalURI = + new URI(mURL + "&signedRequest=" + URLEncoder.encode(new String(mDrmRequest), "UTF-8")); + urlConnection = (HttpURLConnection) ProxySelector.openConnectionWithProxy(finalURI); + urlConnection.setRequestMethod("POST"); + if (DEBUG) Log.d(LOGTAG, "Provisioning, posting url =" + finalURI.toString()); + + // Add data + urlConnection.setRequestProperty("Accept", "*/*"); + urlConnection.setRequestProperty("User-Agent", getCDMUserAgent()); + urlConnection.setRequestProperty("Content-Type", "application/json"); + + // Execute HTTP Post Request + urlConnection.connect(); + + final int responseCode = urlConnection.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { + in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), UTF_8)); + String inputLine; + final StringBuffer response = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + mResponseBody = String.valueOf(response).getBytes(UTF_8); + if (DEBUG) Log.d(LOGTAG, "Provisioning, response received."); + if (mResponseBody != null) Log.d(LOGTAG, "response length=" + mResponseBody.length); + } else { + Log.d(LOGTAG, "Provisioning, server returned HTTP error code :" + responseCode); + } + } catch (final IOException e) { + Log.e(LOGTAG, "Got exception during posting provisioning request ...", e); + } catch (final URISyntaxException e) { + Log.e(LOGTAG, "Got exception during creating uri ...", e); + } finally { + if (urlConnection != null) { + urlConnection.disconnect(); + } + try { + if (in != null) { + in.close(); + } + } catch (final IOException e) { + Log.e(LOGTAG, "Exception during closing in ...", e); + } + } + return null; + } + + @Override + protected void onPostExecute(final Void v) { + onProvisionResponse(mPromiseId, mResponseBody); + } + } + + private boolean provideProvisionResponse(final byte[] response) { + if (response == null || response.length == 0) { + if (DEBUG) Log.d(LOGTAG, "Invalid provision response."); + return false; + } + + try { + mDrm.provideProvisionResponse(response); + return true; + } catch (final android.media.DeniedByServerException e) { + if (DEBUG) Log.d(LOGTAG, "Failed to provide provision response:" + e.getMessage()); + } catch (final java.lang.IllegalStateException e) { + if (DEBUG) Log.d(LOGTAG, "Failed to provide provision response:" + e.getMessage()); + } + return false; + } + + private void savePendingCreateSessionData( + final int token, final int promiseId, final byte[] initData, final String mime) { + if (DEBUG) Log.d(LOGTAG, "savePendingCreateSessionData, promiseId : " + promiseId); + mPendingCreateSessionDataQueue.offer( + new PendingCreateSessionData(token, promiseId, initData, mime)); + } + + private void processPendingCreateSessionData() { + if (DEBUG) Log.d(LOGTAG, "processPendingCreateSessionData ... "); + + assertTrue(mProvisioningPromiseId == 0); + try { + while (!mPendingCreateSessionDataQueue.isEmpty()) { + final PendingCreateSessionData pendingData = mPendingCreateSessionDataQueue.poll(); + if (pendingData == null) { + return; + } + if (DEBUG) + Log.d(LOGTAG, "processPendingCreateSessionData, promiseId : " + pendingData.mPromiseId); + + createSession( + pendingData.mToken, + pendingData.mPromiseId, + pendingData.mMimeType, + pendingData.mInitData); + } + } catch (final Exception e) { + Log.e(LOGTAG, "Got excpetion during processPendingCreateSessionData ...", e); + } + } + + private void resumePendingOperations() { + if (mHandlerThread == null) { + mHandlerThread = new HandlerThread("PendingSessionOpsThread"); + mHandlerThread.start(); + } + if (mHandler == null) { + mHandler = new Handler(mHandlerThread.getLooper()); + } + mHandler.post( + new Runnable() { + @Override + public void run() { + if (mPendingKeyRequest != null) { + MediaDrm.KeyRequest request = null; + try { + request = + getKeyRequest( + mPendingKeyRequest.mSession, + mPendingKeyRequest.mData, + mPendingKeyRequest.mMimeType); + } catch (final NotProvisionedException e) { + Log.e(LOGTAG, "Cannot get key request after provisioning!"); + return; + } finally { + mPendingKeyRequest = null; + } + requestLicense(mPendingKeyRequest.mSession.array(), request); + } else { + processPendingCreateSessionData(); + } + } + }); + } + + private void requestLicense(final byte[] session, final MediaDrm.KeyRequest request) { + if (request == null) { + Log.e(LOGTAG, "null key request when requesting license"); + return; + } + // The EME spec says the messageType is only for optimization and optional. + // Send 'License_request' as default when it's not available. + int requestType = LICENSE_REQUEST_INITIAL; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + requestType = request.getRequestType(); + } + onSessionMessage(session, requestType, request.getData()); + } + + // Only triggered when failed on {openSession, getKeyRequest} + private void startProvisioning(final int promiseId) { + if (DEBUG) Log.d(LOGTAG, "startProvisioning()"); + if (mProvisioningPromiseId > 0) { + // Already in provisioning. + return; + } + try { + mProvisioningPromiseId = promiseId; + final MediaDrm.ProvisionRequest request = mDrm.getProvisionRequest(); + mProvisionTask = new PostRequestTask(promiseId, request.getDefaultUrl(), request.getData()); + mProvisionTask.execute(); + } catch (final Exception e) { + onRejectPromise(promiseId, "Exception happened in startProvisioning !"); + mProvisioningPromiseId = 0; + } + } + + private void onProvisionResponse(final int promiseId, final byte[] response) { + if (DEBUG) Log.d(LOGTAG, "onProvisionResponse()"); + mProvisionTask = null; + mProvisioningPromiseId = 0; + final boolean success = provideProvisionResponse(response); + if (success) { + // Promise will either be resovled / rejected in createSession during + // resuming operations. + resumePendingOperations(); + } else { + onRejectPromise(promiseId, "Failed to provide provision response."); + } + } + + private boolean ensureMediaCryptoCreated() throws android.media.NotProvisionedException { + if (mCrypto != null) { + return true; + } + try { + mCryptoSessionId = openSession(); + if (mCryptoSessionId == null) { + if (DEBUG) Log.d(LOGTAG, "Cannot open session for MediaCrypto"); + return false; + } + + if (MediaCrypto.isCryptoSchemeSupported(mSchemeUUID)) { + final byte[] cryptoSessionId = mCryptoSessionId.array(); + mCrypto = new MediaCrypto(mSchemeUUID, cryptoSessionId); + mSessionIds.add(mCryptoSessionId); + if (DEBUG) + Log.d( + LOGTAG, + "MediaCrypto successfully created! - SId " + + INVALID_SESSION_ID + + ", " + + new String(cryptoSessionId, UTF_8)); + return true; + } else { + if (DEBUG) Log.d(LOGTAG, "Cannot create MediaCrypto for unsupported scheme."); + return false; + } + } catch (final android.media.MediaCryptoException e) { + if (DEBUG) Log.d(LOGTAG, "Cannot create MediaCrypto:" + e.getMessage()); + release(); + return false; + } catch (final android.media.NotProvisionedException e) { + if (DEBUG) + Log.d(LOGTAG, "ensureMediaCryptoCreated::Device not provisioned:" + e.getMessage()); + throw e; + } + } + + private UUID convertKeySystemToSchemeUUID(final String keySystem) { + if (WIDEVINE_KEY_SYSTEM.equals(keySystem)) { + return WIDEVINE_SCHEME_UUID; + } + if (DEBUG) Log.d(LOGTAG, "Cannot convert unsupported key system : " + keySystem); + return new UUID(0L, 0L); + } + + private String getCDMUserAgent() { + // This user agent is found and hard-coded in Android(L) source code and + // Chromium project. Not sure if it's gonna change in the future. + return "Widevine CDM v1.0"; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV23.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV23.java new file mode 100644 index 0000000000..bee2635a81 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoMediaDrmBridgeV23.java @@ -0,0 +1,50 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import static android.os.Build.VERSION_CODES.M; + +import android.annotation.TargetApi; +import android.media.MediaDrm; +import android.util.Log; +import java.util.List; + +@TargetApi(M) +public class GeckoMediaDrmBridgeV23 extends GeckoMediaDrmBridgeV21 { + private static final boolean DEBUG = false; + + GeckoMediaDrmBridgeV23(final String keySystem) throws Exception { + super(keySystem); + if (DEBUG) Log.d(LOGTAG, "GeckoMediaDrmBridgeV23 ctor"); + mDrm.setOnKeyStatusChangeListener(new KeyStatusChangeListener(), null); + } + + private class KeyStatusChangeListener implements MediaDrm.OnKeyStatusChangeListener { + @Override + public void onKeyStatusChange( + final MediaDrm mediaDrm, + final byte[] sessionId, + final List<MediaDrm.KeyStatus> keyInformation, + final boolean hasNewUsableKey) { + if (DEBUG) Log.d(LOGTAG, "[onKeyStatusChange] hasNewUsableKey = " + hasNewUsableKey); + if (keyInformation.size() == 0) { + return; + } + final SessionKeyInfo[] keyInfos = new SessionKeyInfo[keyInformation.size()]; + for (int i = 0; i < keyInformation.size(); i++) { + final MediaDrm.KeyStatus keyStatus = keyInformation.get(i); + keyInfos[i] = new SessionKeyInfo(keyStatus.getKeyId(), keyStatus.getStatusCode()); + } + onSessionBatchedKeyChanged(sessionId, keyInfos); + if (DEBUG) Log.d(LOGTAG, "Key successfully added for session " + new String(sessionId)); + } + } + + @Override + protected void HandleKeyStatusChangeByDummyKey(final String sessionId) { + // MediaDrm.KeyStatus information listener is supported on M+, there is no need to use + // dummy key id to report key status anymore. + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoPlayerFactory.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoPlayerFactory.java new file mode 100644 index 0000000000..47278115d3 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoPlayerFactory.java @@ -0,0 +1,43 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.util.Log; +import androidx.annotation.NonNull; +import java.util.ArrayList; + +public final class GeckoPlayerFactory { + public static final ArrayList<BaseHlsPlayer> sPlayerList = new ArrayList<BaseHlsPlayer>(); + + static synchronized BaseHlsPlayer getPlayer() { + try { + final Class<?> cls = Class.forName("org.mozilla.gecko.media.GeckoHlsPlayer"); + final BaseHlsPlayer player = (BaseHlsPlayer) cls.newInstance(); + sPlayerList.add(player); + return player; + } catch (final Exception e) { + Log.e("GeckoPlayerFactory", "Class GeckoHlsPlayer not found or failed to create", e); + } + return null; + } + + static synchronized BaseHlsPlayer getPlayer(final int id) { + for (final BaseHlsPlayer player : sPlayerList) { + if (player.getId() == id) { + return player; + } + } + Log.w("GeckoPlayerFactory", "No player found with id : " + id); + return null; + } + + static synchronized void removePlayer(final @NonNull BaseHlsPlayer player) { + final int index = sPlayerList.indexOf(player); + if (index >= 0) { + sPlayerList.remove(player); + Log.d("GeckoPlayerFactory", "HlsPlayer with id(" + player.getId() + ") is removed."); + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoVideoInfo.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoVideoInfo.java new file mode 100644 index 0000000000..c641c58354 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/GeckoVideoInfo.java @@ -0,0 +1,45 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import org.mozilla.gecko.annotation.WrapForJNI; + +// A subset of the class VideoInfo in dom/media/MediaInfo.h +@WrapForJNI +public final class GeckoVideoInfo { + public final byte[] codecSpecificData; + public final byte[] extraData; + public final int displayWidth; + public final int displayHeight; + public final int pictureWidth; + public final int pictureHeight; + public final int rotation; + public final int stereoMode; + public final long duration; + public final String mimeType; + + public GeckoVideoInfo( + final int displayWidth, + final int displayHeight, + final int pictureWidth, + final int pictureHeight, + final int rotation, + final int stereoMode, + final long duration, + final String mimeType, + final byte[] extraData, + final byte[] codecSpecificData) { + this.displayWidth = displayWidth; + this.displayHeight = displayHeight; + this.pictureWidth = pictureWidth; + this.pictureHeight = pictureHeight; + this.rotation = rotation; + this.stereoMode = stereoMode; + this.duration = duration; + this.mimeType = mimeType; + this.extraData = extraData; + this.codecSpecificData = codecSpecificData; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/JellyBeanAsyncCodec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/JellyBeanAsyncCodec.java new file mode 100644 index 0000000000..3b055f0bca --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/JellyBeanAsyncCodec.java @@ -0,0 +1,481 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.os.Message; +import android.util.Log; +import android.view.Surface; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.mozilla.gecko.util.HardwareCodecCapabilityUtils; + +// Implement async API using MediaCodec sync mode (API v16). +// This class uses internal worker thread/handler (mBufferPoller) to poll +// input and output buffer and notifies the client through callbacks. +final class JellyBeanAsyncCodec implements AsyncCodec { + private static final String LOGTAG = "GeckoAsyncCodecAPIv16"; + private static final boolean DEBUG = false; + + private static final int ERROR_CODEC = -10000; + + private abstract class CancelableHandler extends Handler { + private static final int MSG_CANCELLATION = 0x434E434C; // 'CNCL' + + protected CancelableHandler(final Looper looper) { + super(looper); + } + + protected void cancel() { + removeCallbacksAndMessages(null); + sendEmptyMessage(MSG_CANCELLATION); + // Wait until handleMessageLocked() is done. + synchronized (this) { + } + } + + protected boolean isCanceled() { + return hasMessages(MSG_CANCELLATION); + } + + // Subclass should implement this and return true if it handles msg. + // Warning: Never, ever call super.handleMessage() in this method! + protected abstract boolean handleMessageLocked(Message msg); + + public final void handleMessage(final Message msg) { + // Block cancel() during handleMessageLocked(). + synchronized (this) { + if (isCanceled() || handleMessageLocked(msg)) { + return; + } + } + + switch (msg.what) { + case MSG_CANCELLATION: + // Just a marker. Nothing to do here. + if (DEBUG) { + Log.d( + LOGTAG, + "handler " + this + " done cancellation, codec=" + JellyBeanAsyncCodec.this); + } + break; + default: + super.handleMessage(msg); + break; + } + } + } + + // A handler to invoke AsyncCodec.Callbacks methods. + private final class CallbackSender extends CancelableHandler { + private static final int MSG_INPUT_BUFFER_AVAILABLE = 1; + private static final int MSG_OUTPUT_BUFFER_AVAILABLE = 2; + private static final int MSG_OUTPUT_FORMAT_CHANGE = 3; + private static final int MSG_ERROR = 4; + private Callbacks mCallbacks; + + private CallbackSender(final Looper looper, final Callbacks callbacks) { + super(looper); + mCallbacks = callbacks; + } + + public void notifyInputBuffer(final int index) { + if (isCanceled()) { + return; + } + + final Message msg = obtainMessage(MSG_INPUT_BUFFER_AVAILABLE); + msg.arg1 = index; + processMessage(msg); + } + + private void processMessage(final Message msg) { + if (Looper.myLooper() == getLooper()) { + handleMessage(msg); + } else { + sendMessage(msg); + } + } + + public void notifyOutputBuffer(final int index, final MediaCodec.BufferInfo info) { + if (isCanceled()) { + return; + } + + final Message msg = obtainMessage(MSG_OUTPUT_BUFFER_AVAILABLE, info); + msg.arg1 = index; + processMessage(msg); + } + + public void notifyOutputFormat(final MediaFormat format) { + if (isCanceled()) { + return; + } + processMessage(obtainMessage(MSG_OUTPUT_FORMAT_CHANGE, format)); + } + + public void notifyError(final int result) { + Log.e(LOGTAG, "codec error:" + result); + processMessage(obtainMessage(MSG_ERROR, result, 0)); + } + + protected boolean handleMessageLocked(final Message msg) { + switch (msg.what) { + case MSG_INPUT_BUFFER_AVAILABLE: // arg1: buffer index. + mCallbacks.onInputBufferAvailable(JellyBeanAsyncCodec.this, msg.arg1); + break; + case MSG_OUTPUT_BUFFER_AVAILABLE: // arg1: buffer index, obj: info. + mCallbacks.onOutputBufferAvailable( + JellyBeanAsyncCodec.this, msg.arg1, (MediaCodec.BufferInfo) msg.obj); + break; + case MSG_OUTPUT_FORMAT_CHANGE: // obj: output format. + mCallbacks.onOutputFormatChanged(JellyBeanAsyncCodec.this, (MediaFormat) msg.obj); + break; + case MSG_ERROR: // arg1: error code. + mCallbacks.onError(JellyBeanAsyncCodec.this, msg.arg1); + break; + default: + return false; + } + + return true; + } + } + + // Handler to poll input and output buffers using dequeue(Input|Output)Buffer(), + // with 10ms time-out. Once triggered and successfully gets a buffer, it + // will schedule next polling until EOS or failure. To prevent it from + // automatically polling more buffer, use cancel() it inherits from + // CancelableHandler. + private final class BufferPoller extends CancelableHandler { + private static final int MSG_POLL_INPUT_BUFFERS = 1; + private static final int MSG_POLL_OUTPUT_BUFFERS = 2; + + private static final long DEQUEUE_TIMEOUT_US = 10000; + + public BufferPoller(final Looper looper) { + super(looper); + } + + private void schedulePollingIfNotCanceled(final int what) { + if (isCanceled()) { + return; + } + + schedulePolling(what); + } + + private void schedulePolling(final int what) { + if (needsBuffer(what)) { + sendEmptyMessage(what); + } + } + + private boolean needsBuffer(final int what) { + if (mOutputEnded && (what == MSG_POLL_OUTPUT_BUFFERS)) { + return false; + } + + return !(mInputEnded && (what == MSG_POLL_INPUT_BUFFERS)); + } + + protected boolean handleMessageLocked(final Message msg) { + try { + switch (msg.what) { + case MSG_POLL_INPUT_BUFFERS: + pollInputBuffer(); + break; + case MSG_POLL_OUTPUT_BUFFERS: + pollOutputBuffer(); + break; + default: + return false; + } + } catch (final IllegalStateException e) { + e.printStackTrace(); + mCallbackSender.notifyError(ERROR_CODEC); + } + + return true; + } + + private void pollInputBuffer() { + final int result = mCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); + if (result >= 0) { + mCallbackSender.notifyInputBuffer(result); + } else if (result == MediaCodec.INFO_TRY_AGAIN_LATER) { + mBufferPoller.schedulePollingIfNotCanceled(BufferPoller.MSG_POLL_INPUT_BUFFERS); + } else { + mCallbackSender.notifyError(result); + } + } + + private void pollOutputBuffer() { + boolean dequeueMoreBuffer = true; + final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + final int result = mCodec.dequeueOutputBuffer(info, DEQUEUE_TIMEOUT_US); + if (result >= 0) { + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + mOutputEnded = true; + } + mCallbackSender.notifyOutputBuffer(result, info); + } else if (result == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + mOutputBuffers = mCodec.getOutputBuffers(); + } else if (result == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + mOutputBuffers = mCodec.getOutputBuffers(); + mCallbackSender.notifyOutputFormat(mCodec.getOutputFormat()); + } else if (result == MediaCodec.INFO_TRY_AGAIN_LATER) { + // When input ended, keep polling remaining output buffer until EOS. + dequeueMoreBuffer = mInputEnded; + } else { + mCallbackSender.notifyError(result); + dequeueMoreBuffer = false; + } + + if (dequeueMoreBuffer) { + schedulePollingIfNotCanceled(MSG_POLL_OUTPUT_BUFFERS); + } + } + } + + private MediaCodec mCodec; + private ByteBuffer[] mInputBuffers; + private ByteBuffer[] mOutputBuffers; + private AsyncCodec.Callbacks mCallbacks; + private CallbackSender mCallbackSender; + + private BufferPoller mBufferPoller; + private volatile boolean mInputEnded; + private volatile boolean mOutputEnded; + + // Must be called on a thread with looper. + /* package */ JellyBeanAsyncCodec(final String name) throws IOException { + mCodec = MediaCodec.createByCodecName(name); + initBufferPoller(name + " buffer poller"); + } + + private void initBufferPoller(final String name) { + if (mBufferPoller != null) { + Log.e(LOGTAG, "poller already initialized"); + return; + } + final HandlerThread thread = new HandlerThread(name); + thread.start(); + mBufferPoller = new BufferPoller(thread.getLooper()); + if (DEBUG) { + Log.d(LOGTAG, "start poller for codec:" + this + ", thread=" + thread.getThreadId()); + } + } + + @Override + public void setCallbacks(final AsyncCodec.Callbacks callbacks, final Handler handler) { + if (callbacks == null) { + return; + } + + Looper looper = (handler == null) ? null : handler.getLooper(); + if (looper == null) { + // Use this thread if no handler supplied. + looper = Looper.myLooper(); + } + if (looper == null) { + // This thread has no looper. Use poller thread. + looper = mBufferPoller.getLooper(); + } + mCallbackSender = new CallbackSender(looper, callbacks); + if (DEBUG) { + Log.d(LOGTAG, "setCallbacks(): sender=" + mCallbackSender); + } + } + + @Override + public void configure( + final MediaFormat format, final Surface surface, final MediaCrypto crypto, final int flags) { + assertCallbacks(); + + mCodec.configure(format, surface, crypto, flags); + } + + @Override + public boolean isAdaptivePlaybackSupported(final String mimeType) { + return HardwareCodecCapabilityUtils.checkSupportsAdaptivePlayback(mCodec, mimeType); + } + + @Override + public boolean isTunneledPlaybackSupported(final String mimeType) { + try { + return mCodec + .getCodecInfo() + .getCapabilitiesForType(mimeType) + .isFeatureSupported(CodecCapabilities.FEATURE_TunneledPlayback); + } catch (final Exception e) { + return false; + } + } + + private void assertCallbacks() { + if (mCallbackSender == null) { + throw new IllegalStateException(LOGTAG + ": callback must be supplied with setCallbacks()."); + } + } + + @Override + public void start() { + assertCallbacks(); + + mCodec.start(); + mInputEnded = false; + mOutputEnded = false; + mInputBuffers = mCodec.getInputBuffers(); + resumeReceivingInputs(); + mOutputBuffers = mCodec.getOutputBuffers(); + } + + @Override + public void resumeReceivingInputs() { + for (int i = 0; i < mInputBuffers.length; i++) { + mBufferPoller.schedulePolling(BufferPoller.MSG_POLL_INPUT_BUFFERS); + } + } + + @Override + public final void setBitrate(final int bps) { + final Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, bps); + mCodec.setParameters(params); + } + + @Override + public final void queueInputBuffer( + final int index, + final int offset, + final int size, + final long presentationTimeUs, + final int flags) { + assertCallbacks(); + + mInputEnded = (flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + + if (((flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0)) { + final Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + mCodec.setParameters(params); + } + + try { + mCodec.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } catch (final IllegalStateException e) { + e.printStackTrace(); + mCallbackSender.notifyError(ERROR_CODEC); + return; + } + + mBufferPoller.schedulePolling(BufferPoller.MSG_POLL_OUTPUT_BUFFERS); + mBufferPoller.schedulePolling(BufferPoller.MSG_POLL_INPUT_BUFFERS); + } + + @Override + public final void queueSecureInputBuffer( + final int index, + final int offset, + final MediaCodec.CryptoInfo cryptoInfo, + final long presentationTimeUs, + final int flags) { + assertCallbacks(); + + mInputEnded = (flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + + try { + mCodec.queueSecureInputBuffer(index, offset, cryptoInfo, presentationTimeUs, flags); + } catch (final IllegalStateException e) { + e.printStackTrace(); + mCallbackSender.notifyError(ERROR_CODEC); + return; + } + + mBufferPoller.schedulePolling(BufferPoller.MSG_POLL_INPUT_BUFFERS); + mBufferPoller.schedulePolling(BufferPoller.MSG_POLL_OUTPUT_BUFFERS); + } + + @Override + public final void releaseOutputBuffer(final int index, final boolean render) { + assertCallbacks(); + + mCodec.releaseOutputBuffer(index, render); + } + + @Override + public final ByteBuffer getInputBuffer(final int index) { + assertCallbacks(); + + return mInputBuffers[index]; + } + + @Override + public final ByteBuffer getOutputBuffer(final int index) { + assertCallbacks(); + + return mOutputBuffers[index]; + } + + @Override + public MediaFormat getInputFormat() { + return null; + } + + @Override + public void flush() { + assertCallbacks(); + + mInputEnded = false; + mOutputEnded = false; + cancelPendingTasks(); + mCodec.flush(); + } + + private void cancelPendingTasks() { + mBufferPoller.cancel(); + mCallbackSender.cancel(); + } + + @Override + public void stop() { + assertCallbacks(); + + cancelPendingTasks(); + mCodec.stop(); + } + + @Override + public void release() { + assertCallbacks(); + + cancelPendingTasks(); + mCallbackSender = null; + mCodec.release(); + stopBufferPoller(); + } + + private void stopBufferPoller() { + if (mBufferPoller == null) { + Log.e(LOGTAG, "no initialized poller."); + return; + } + + mBufferPoller.getLooper().quit(); + mBufferPoller = null; + + if (DEBUG) { + Log.d(LOGTAG, "stop poller " + this); + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/LollipopAsyncCodec.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/LollipopAsyncCodec.java new file mode 100644 index 0000000000..aaf8810bbb --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/LollipopAsyncCodec.java @@ -0,0 +1,248 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.view.Surface; +import androidx.annotation.NonNull; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.mozilla.gecko.util.HardwareCodecCapabilityUtils; + +/* package */ final class LollipopAsyncCodec implements AsyncCodec { + private final MediaCodec mCodec; + + private class CodecCallback extends MediaCodec.Callback { + private final Forwarder mForwarder; + + private class Forwarder extends Handler { + private static final int MSG_INPUT_BUFFER_AVAILABLE = 1; + private static final int MSG_OUTPUT_BUFFER_AVAILABLE = 2; + private static final int MSG_OUTPUT_FORMAT_CHANGE = 3; + private static final int MSG_ERROR = 4; + + private final Callbacks mTarget; + + private Forwarder(final Looper looper, final Callbacks target) { + super(looper); + mTarget = target; + } + + @Override + public void handleMessage(final Message msg) { + switch (msg.what) { + case MSG_INPUT_BUFFER_AVAILABLE: + mTarget.onInputBufferAvailable(LollipopAsyncCodec.this, msg.arg1); // index + break; + case MSG_OUTPUT_BUFFER_AVAILABLE: + mTarget.onOutputBufferAvailable( + LollipopAsyncCodec.this, + msg.arg1, // index + (MediaCodec.BufferInfo) msg.obj); // buffer info + break; + case MSG_OUTPUT_FORMAT_CHANGE: + mTarget.onOutputFormatChanged( + LollipopAsyncCodec.this, (MediaFormat) msg.obj); // output format + break; + case MSG_ERROR: + mTarget.onError(LollipopAsyncCodec.this, msg.arg1); // error code + break; + default: + super.handleMessage(msg); + } + } + + private void onInput(final int index) { + notify(obtainMessage(MSG_INPUT_BUFFER_AVAILABLE, index, 0)); + } + + private void notify(final Message msg) { + if (Looper.myLooper() == getLooper()) { + handleMessage(msg); + } else { + sendMessage(msg); + } + } + + private void onOutput(final int index, final MediaCodec.BufferInfo info) { + final Message msg = obtainMessage(MSG_OUTPUT_BUFFER_AVAILABLE, index, 0, info); + notify(msg); + } + + private void onOutputFormatChanged(final MediaFormat format) { + notify(obtainMessage(MSG_OUTPUT_FORMAT_CHANGE, format)); + } + + private void onError(final MediaCodec.CodecException e) { + e.printStackTrace(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + notify(obtainMessage(MSG_ERROR, e.getErrorCode())); + } else { + notify(obtainMessage(MSG_ERROR, e.getLocalizedMessage())); + } + } + } + + private CodecCallback(final Callbacks callbacks, final Handler handler) { + Looper looper = (handler == null) ? null : handler.getLooper(); + if (looper == null) { + // Use this thread if no handler supplied. + looper = Looper.myLooper(); + } + if (looper == null) { + // This thread has no looper. Use main thread. + looper = Looper.getMainLooper(); + } + + mForwarder = new Forwarder(looper, callbacks); + } + + @Override + public void onInputBufferAvailable(@NonNull final MediaCodec codec, final int index) { + mForwarder.onInput(index); + } + + @Override + public void onOutputBufferAvailable( + @NonNull final MediaCodec codec, + final int index, + @NonNull final MediaCodec.BufferInfo info) { + mForwarder.onOutput(index, info); + } + + @Override + public void onOutputFormatChanged( + @NonNull final MediaCodec codec, @NonNull final MediaFormat format) { + mForwarder.onOutputFormatChanged(format); + } + + @Override + public void onError( + @NonNull final MediaCodec codec, @NonNull final MediaCodec.CodecException e) { + mForwarder.onError(e); + } + } + + /* package */ LollipopAsyncCodec(final String name) throws IOException { + mCodec = MediaCodec.createByCodecName(name); + } + + @Override + public void setCallbacks(final Callbacks callbacks, final Handler handler) { + if (callbacks == null) { + return; + } + + mCodec.setCallback(new CodecCallback(callbacks, handler)); + } + + @Override + public void configure( + final MediaFormat format, final Surface surface, final MediaCrypto crypto, final int flags) { + mCodec.configure(format, surface, crypto, flags); + } + + @Override + public boolean isAdaptivePlaybackSupported(final String mimeType) { + return HardwareCodecCapabilityUtils.checkSupportsAdaptivePlayback(mCodec, mimeType); + } + + @Override + public boolean isTunneledPlaybackSupported(final String mimeType) { + try { + return mCodec + .getCodecInfo() + .getCapabilitiesForType(mimeType) + .isFeatureSupported(CodecCapabilities.FEATURE_TunneledPlayback); + } catch (final Exception e) { + return false; + } + } + + @Override + public void start() { + mCodec.start(); + } + + @Override + public void stop() { + mCodec.stop(); + } + + @Override + public void flush() { + mCodec.flush(); + } + + @Override + public void resumeReceivingInputs() { + mCodec.start(); + } + + @Override + public void setBitrate(final int bps) { + final Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, bps); + mCodec.setParameters(params); + } + + @Override + public void release() { + mCodec.release(); + } + + @Override + public ByteBuffer getInputBuffer(final int index) { + return mCodec.getInputBuffer(index); + } + + @Override + public ByteBuffer getOutputBuffer(final int index) { + return mCodec.getOutputBuffer(index); + } + + @Override + public MediaFormat getInputFormat() { + return mCodec.getInputFormat(); + } + + @Override + public void queueInputBuffer( + final int index, + final int offset, + final int size, + final long presentationTimeUs, + final int flags) { + if ((flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { + final Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + mCodec.setParameters(params); + } + mCodec.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + + @Override + public void queueSecureInputBuffer( + final int index, + final int offset, + final MediaCodec.CryptoInfo info, + final long presentationTimeUs, + final int flags) { + mCodec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); + } + + @Override + public void releaseOutputBuffer(final int index, final boolean render) { + mCodec.releaseOutputBuffer(index, render); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaDrmProxy.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaDrmProxy.java new file mode 100644 index 0000000000..1bfab37063 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaDrmProxy.java @@ -0,0 +1,297 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.annotation.SuppressLint; +import android.media.MediaCrypto; +import android.media.MediaDrm; +import android.os.Build; +import android.util.Log; +import java.util.ArrayList; +import java.util.UUID; +import org.mozilla.gecko.annotation.WrapForJNI; +import org.mozilla.gecko.mozglue.JNIObject; + +public final class MediaDrmProxy { + private static final String LOGTAG = "GeckoMediaDrmProxy"; + private static final boolean DEBUG = false; + private static final UUID WIDEVINE_SCHEME_UUID = + new UUID(0xedef8ba979d64aceL, 0xa3c827dcd51d21edL); + + private static final String WIDEVINE_KEY_SYSTEM = "com.widevine.alpha"; + @WrapForJNI private static final String AAC = "audio/mp4a-latm"; + @WrapForJNI private static final String AVC = "video/avc"; + @WrapForJNI private static final String VORBIS = "audio/vorbis"; + @WrapForJNI private static final String VP8 = "video/x-vnd.on2.vp8"; + @WrapForJNI private static final String VP9 = "video/x-vnd.on2.vp9"; + @WrapForJNI private static final String OPUS = "audio/opus"; + @WrapForJNI private static final String FLAC = "audio/flac"; + + public static final ArrayList<MediaDrmProxy> sProxyList = new ArrayList<MediaDrmProxy>(); + + // A flag to avoid using the native object that has been destroyed. + private boolean mDestroyed; + private GeckoMediaDrm mImpl; + private String mDrmStubId; + + private static boolean isSystemSupported() { + // Support versions >= Marshmallow + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + if (DEBUG) + Log.d(LOGTAG, "System Not supported !!, current SDK version is " + Build.VERSION.SDK_INT); + return false; + } + return true; + } + + @SuppressLint("NewApi") + @WrapForJNI + public static boolean isSchemeSupported(final String keySystem) { + if (!isSystemSupported()) { + return false; + } + if (keySystem.equals(WIDEVINE_KEY_SYSTEM)) { + return MediaDrm.isCryptoSchemeSupported(WIDEVINE_SCHEME_UUID) + && MediaCrypto.isCryptoSchemeSupported(WIDEVINE_SCHEME_UUID); + } + if (DEBUG) Log.d(LOGTAG, "isSchemeSupported key sytem = " + keySystem); + return false; + } + + @SuppressLint("NewApi") + @WrapForJNI + public static boolean IsCryptoSchemeSupported(final String keySystem, final String container) { + if (!isSystemSupported()) { + return false; + } + if (keySystem.equals(WIDEVINE_KEY_SYSTEM)) { + return MediaDrm.isCryptoSchemeSupported(WIDEVINE_SCHEME_UUID, container); + } + if (DEBUG) + Log.d(LOGTAG, "cannot decrypt key sytem = " + keySystem + ", container = " + container); + return false; + } + + // Interface for callback to native. + public interface Callbacks { + void onSessionCreated(int createSessionToken, int promiseId, byte[] sessionId, byte[] request); + + void onSessionUpdated(int promiseId, byte[] sessionId); + + void onSessionClosed(int promiseId, byte[] sessionId); + + void onSessionMessage(byte[] sessionId, int sessionMessageType, byte[] request); + + void onSessionError(byte[] sessionId, String message); + + // MediaDrm.KeyStatus is available in API level 23(M) + // https://developer.android.com/reference/android/media/MediaDrm.KeyStatus.html + // For compatibility between L and M above, we'll unwrap the KeyStatus structure + // and store the keyid and status into SessionKeyInfo and pass to native(MediaDrmCDMProxy). + void onSessionBatchedKeyChanged(byte[] sessionId, SessionKeyInfo[] keyInfos); + + void onRejectPromise(int promiseId, String message); + } // Callbacks + + public static class NativeMediaDrmProxyCallbacks extends JNIObject implements Callbacks { + @WrapForJNI(calledFrom = "gecko") + NativeMediaDrmProxyCallbacks() {} + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionCreated( + int createSessionToken, int promiseId, byte[] sessionId, byte[] request); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionUpdated(int promiseId, byte[] sessionId); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionClosed(int promiseId, byte[] sessionId); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionMessage(byte[] sessionId, int sessionMessageType, byte[] request); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionError(byte[] sessionId, String message); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onSessionBatchedKeyChanged(byte[] sessionId, SessionKeyInfo[] keyInfos); + + @Override + @WrapForJNI(dispatchTo = "gecko") + public native void onRejectPromise(int promiseId, String message); + + @Override // JNIObject + protected void disposeNative() { + throw new UnsupportedOperationException(); + } + } // NativeMediaDrmProxyCallbacks + + // A proxy to callback from LocalMediaDrmBridge to native instance. + public static class MediaDrmProxyCallbacks implements GeckoMediaDrm.Callbacks { + private final Callbacks mNativeCallbacks; + private final MediaDrmProxy mProxy; + + public MediaDrmProxyCallbacks(final MediaDrmProxy proxy, final Callbacks callbacks) { + mNativeCallbacks = callbacks; + mProxy = proxy; + } + + @Override + public void onSessionCreated( + final int createSessionToken, + final int promiseId, + final byte[] sessionId, + final byte[] request) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionCreated(createSessionToken, promiseId, sessionId, request); + } + } + + @Override + public void onSessionUpdated(final int promiseId, final byte[] sessionId) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionUpdated(promiseId, sessionId); + } + } + + @Override + public void onSessionClosed(final int promiseId, final byte[] sessionId) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionClosed(promiseId, sessionId); + } + } + + @Override + public void onSessionMessage( + final byte[] sessionId, final int sessionMessageType, final byte[] request) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionMessage(sessionId, sessionMessageType, request); + } + } + + @Override + public void onSessionError(final byte[] sessionId, final String message) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionError(sessionId, message); + } + } + + @Override + public void onSessionBatchedKeyChanged( + final byte[] sessionId, final SessionKeyInfo[] keyInfos) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onSessionBatchedKeyChanged(sessionId, keyInfos); + } + } + + @Override + public void onRejectPromise(final int promiseId, final String message) { + if (!mProxy.isDestroyed()) { + mNativeCallbacks.onRejectPromise(promiseId, message); + } + } + } // MediaDrmProxyCallbacks + + public boolean isDestroyed() { + return mDestroyed; + } + + @WrapForJNI(calledFrom = "gecko") + public static MediaDrmProxy create(final String keySystem, final Callbacks nativeCallbacks) { + return new MediaDrmProxy(keySystem, nativeCallbacks); + } + + MediaDrmProxy(final String keySystem, final Callbacks nativeCallbacks) { + if (DEBUG) Log.d(LOGTAG, "Constructing MediaDrmProxy"); + try { + mDrmStubId = UUID.randomUUID().toString(); + final IMediaDrmBridge remoteBridge = + RemoteManager.getInstance().createRemoteMediaDrmBridge(keySystem, mDrmStubId); + mImpl = new RemoteMediaDrmBridge(remoteBridge); + mImpl.setCallbacks(new MediaDrmProxyCallbacks(this, nativeCallbacks)); + sProxyList.add(this); + } catch (final Exception e) { + Log.e(LOGTAG, "Constructing MediaDrmProxy ... error", e); + } + } + + @WrapForJNI + private void createSession( + final int createSessionToken, + final int promiseId, + final String initDataType, + final byte[] initData) { + if (DEBUG) Log.d(LOGTAG, "createSession, promiseId = " + promiseId); + mImpl.createSession(createSessionToken, promiseId, initDataType, initData); + } + + @WrapForJNI + private void updateSession(final int promiseId, final String sessionId, final byte[] response) { + if (DEBUG) + Log.d(LOGTAG, "updateSession, primiseId(" + promiseId + "sessionId(" + sessionId + ")"); + mImpl.updateSession(promiseId, sessionId, response); + } + + @WrapForJNI + private void closeSession(final int promiseId, final String sessionId) { + if (DEBUG) + Log.d(LOGTAG, "closeSession, primiseId(" + promiseId + "sessionId(" + sessionId + ")"); + mImpl.closeSession(promiseId, sessionId); + } + + @WrapForJNI(calledFrom = "gecko") + private String getStubId() { + return mDrmStubId; + } + + @WrapForJNI + public boolean setServerCertificate(final byte[] cert) { + try { + mImpl.setServerCertificate(cert); + return true; + } catch (final RuntimeException e) { + return false; + } + } + + // Get corresponding MediaCrypto object by a generated UUID for MediaCodec. + // Will be called on MediaFormatReader's TaskQueue. + @WrapForJNI + public static MediaCrypto getMediaCrypto(final String stubId) { + for (final MediaDrmProxy proxy : sProxyList) { + if (proxy.getStubId().equals(stubId)) { + return proxy.getMediaCryptoFromBridge(); + } + } + if (DEBUG) Log.d(LOGTAG, " NULL crytpo "); + return null; + } + + @WrapForJNI // Called when natvie object is destroyed. + private void destroy() { + if (DEBUG) Log.d(LOGTAG, "destroy!! Native object is destroyed."); + if (mDestroyed) { + return; + } + mDestroyed = true; + release(); + } + + private void release() { + if (DEBUG) Log.d(LOGTAG, "release"); + sProxyList.remove(this); + mImpl.release(); + } + + private MediaCrypto getMediaCryptoFromBridge() { + return mImpl != null ? mImpl.getMediaCrypto() : null; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaManager.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaManager.java new file mode 100644 index 0000000000..ef4fdc6932 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/MediaManager.java @@ -0,0 +1,79 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.app.Service; +import android.content.Intent; +import android.os.Binder; +import android.os.IBinder; +import android.os.Process; +import android.os.RemoteException; +import android.util.Log; +import org.mozilla.gecko.mozglue.GeckoLoader; +import org.mozilla.geckoview.BuildConfig; + +public final class MediaManager extends Service { + private static final String LOGTAG = "GeckoMediaManager"; + private static final boolean DEBUG = !BuildConfig.MOZILLA_OFFICIAL; + private static boolean sNativeLibLoaded; + private int mNumActiveRequests = 0; + + private Binder mBinder = + new IMediaManager.Stub() { + @Override + public ICodec createCodec() throws RemoteException { + if (DEBUG) Log.d(LOGTAG, "request codec. Current active requests:" + mNumActiveRequests); + mNumActiveRequests++; + return new Codec(); + } + + @Override + public IMediaDrmBridge createRemoteMediaDrmBridge( + final String keySystem, final String stubId) throws RemoteException { + if (DEBUG) + Log.d(LOGTAG, "request DRM bridge. Current active requests:" + mNumActiveRequests); + mNumActiveRequests++; + return new RemoteMediaDrmBridgeStub(keySystem, stubId); + } + + @Override + public void endRequest() { + if (DEBUG) Log.d(LOGTAG, "end request. Current active requests:" + mNumActiveRequests); + if (mNumActiveRequests > 0) { + mNumActiveRequests--; + } else { + final RuntimeException e = + new RuntimeException("unmatched codec/DRM bridge creation and ending calls!"); + Log.e(LOGTAG, "Error:", e); + } + } + }; + + @Override + public synchronized void onCreate() { + if (!sNativeLibLoaded) { + GeckoLoader.doLoadLibrary(this, "mozglue"); + GeckoLoader.suppressCrashDialog(); + sNativeLibLoaded = true; + } + } + + @Override + public IBinder onBind(final Intent intent) { + return mBinder; + } + + @Override + public boolean onUnbind(final Intent intent) { + Log.i(LOGTAG, "Media service has been unbound. Stopping."); + stopSelf(); + if (mNumActiveRequests != 0) { + // Not unbound by RemoteManager -- caller process is dead. + Log.w(LOGTAG, "unbound while client still active."); + Process.killProcess(Process.myPid()); + } + return false; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteManager.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteManager.java new file mode 100644 index 0000000000..7a2e74c9af --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteManager.java @@ -0,0 +1,248 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.ServiceConnection; +import android.media.MediaFormat; +import android.os.DeadObjectException; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.gfx.GeckoSurface; + +public final class RemoteManager implements IBinder.DeathRecipient { + private static final String LOGTAG = "GeckoRemoteManager"; + private static final boolean DEBUG = false; + private static RemoteManager sRemoteManager = null; + + public static synchronized RemoteManager getInstance() { + if (sRemoteManager == null) { + sRemoteManager = new RemoteManager(); + } + + sRemoteManager.init(); + return sRemoteManager; + } + + private List<CodecProxy> mCodecs = new LinkedList<CodecProxy>(); + private List<IMediaDrmBridge> mDrmBridges = new LinkedList<IMediaDrmBridge>(); + + private volatile IMediaManager mRemote; + + private final class RemoteConnection implements ServiceConnection { + @Override + public void onServiceConnected(final ComponentName name, final IBinder service) { + if (DEBUG) Log.d(LOGTAG, "service connected"); + try { + service.linkToDeath(RemoteManager.this, 0); + } catch (final RemoteException e) { + e.printStackTrace(); + } + synchronized (this) { + mRemote = IMediaManager.Stub.asInterface(service); + notify(); + } + } + + @Override + public void onServiceDisconnected(final ComponentName name) { + if (DEBUG) Log.d(LOGTAG, "service disconnected"); + unlink(); + } + + private boolean connect() { + final Context appCtxt = GeckoAppShell.getApplicationContext(); + appCtxt.bindService( + new Intent(appCtxt, MediaManager.class), + mConnection, + Context.BIND_AUTO_CREATE | Context.BIND_IMPORTANT); + waitConnect(); + return mRemote != null; + } + + // Wait up to 5s. + private synchronized void waitConnect() { + int waitCount = 0; + while (mRemote == null && waitCount < 5) { + try { + wait(1000); + waitCount++; + } catch (final InterruptedException e) { + if (DEBUG) { + e.printStackTrace(); + } + } + } + if (DEBUG) { + Log.d( + LOGTAG, + "wait ~" + waitCount + "s for connection: " + (mRemote == null ? "fail" : "ok")); + } + } + + private synchronized void waitDisconnect() { + while (mRemote != null) { + try { + wait(1000); + } catch (final InterruptedException e) { + if (DEBUG) { + e.printStackTrace(); + } + } + } + } + + private synchronized void unlink() { + if (mRemote == null) { + return; + } + try { + mRemote.asBinder().unlinkToDeath(RemoteManager.this, 0); + } catch (final NoSuchElementException e) { + Log.w(LOGTAG, "death recipient already released"); + } + mRemote = null; + notify(); + } + } + + RemoteConnection mConnection = new RemoteConnection(); + + private synchronized boolean init() { + if (mRemote != null) { + return true; + } + + if (DEBUG) Log.d(LOGTAG, "init remote manager " + this); + return mConnection.connect(); + } + + public synchronized CodecProxy createCodec( + final boolean isEncoder, + final MediaFormat format, + final GeckoSurface surface, + final CodecProxy.Callbacks callbacks, + final String drmStubId) { + if (mRemote == null) { + if (DEBUG) Log.d(LOGTAG, "createCodec failed due to not initialize"); + return null; + } + try { + final ICodec remote = mRemote.createCodec(); + final CodecProxy proxy = + CodecProxy.createCodecProxy(isEncoder, format, surface, callbacks, drmStubId); + if (proxy.init(remote)) { + mCodecs.add(proxy); + return proxy; + } else { + return null; + } + } catch (final RemoteException e) { + e.printStackTrace(); + return null; + } + } + + public synchronized IMediaDrmBridge createRemoteMediaDrmBridge( + final String keySystem, final String stubId) { + if (mRemote == null) { + if (DEBUG) Log.d(LOGTAG, "createRemoteMediaDrmBridge failed due to not initialize"); + return null; + } + try { + final IMediaDrmBridge remoteBridge = mRemote.createRemoteMediaDrmBridge(keySystem, stubId); + mDrmBridges.add(remoteBridge); + return remoteBridge; + } catch (final RemoteException e) { + Log.e(LOGTAG, "Got exception during createRemoteMediaDrmBridge().", e); + return null; + } + } + + @Override + public void binderDied() { + Log.e(LOGTAG, "remote codec is dead"); + handleRemoteDeath(); + } + + private synchronized void handleRemoteDeath() { + mConnection.waitDisconnect(); + + notifyError(!(init() && recoverRemoteCodec())); + } + + private synchronized void notifyError(final boolean fatal) { + for (final CodecProxy proxy : mCodecs) { + proxy.reportError(fatal); + } + } + + private synchronized boolean recoverRemoteCodec() { + if (DEBUG) Log.d(LOGTAG, "recover codec"); + boolean ok = true; + try { + for (final CodecProxy proxy : mCodecs) { + ok &= proxy.init(mRemote.createCodec()); + } + return ok; + } catch (final RemoteException e) { + return false; + } + } + + public void releaseCodec(final CodecProxy proxy) throws DeadObjectException, RemoteException { + if (mRemote == null) { + if (DEBUG) Log.d(LOGTAG, "releaseCodec called but not initialized yet"); + return; + } + proxy.deinit(); + synchronized (this) { + if (mCodecs.remove(proxy)) { + try { + mRemote.endRequest(); + releaseIfNeeded(); + } catch (final RemoteException | NullPointerException e) { + Log.e(LOGTAG, "fail to report remote codec disconnection"); + } + } + } + } + + private void releaseIfNeeded() { + if (!mCodecs.isEmpty() || !mDrmBridges.isEmpty()) { + return; + } + + if (DEBUG) Log.d(LOGTAG, "release remote manager " + this); + mConnection.unlink(); + final Context appCtxt = GeckoAppShell.getApplicationContext(); + appCtxt.unbindService(mConnection); + } + + public void onRemoteMediaDrmBridgeReleased(final IMediaDrmBridge remote) { + if (!mDrmBridges.contains(remote)) { + Log.e(LOGTAG, "Try to release unknown remote MediaDrm bridge: " + remote); + return; + } + + synchronized (this) { + if (mDrmBridges.remove(remote)) { + try { + mRemote.endRequest(); + releaseIfNeeded(); + } catch (final RemoteException | NullPointerException e) { + Log.e(LOGTAG, "Fail to report remote DRM bridge disconnection"); + } + } + } + } +} // RemoteManager diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridge.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridge.java new file mode 100644 index 0000000000..b90f720300 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridge.java @@ -0,0 +1,163 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCrypto; +import android.util.Log; + +final class RemoteMediaDrmBridge implements GeckoMediaDrm { + private static final String LOGTAG = "RemoteMediaDrmBridge"; + private static final boolean DEBUG = false; + private CallbacksForwarder mCallbacksFwd; + private IMediaDrmBridge mRemote; + + // Forward callbacks from remote bridge stub to MediaDrmProxy. + private static class CallbacksForwarder extends IMediaDrmBridgeCallbacks.Stub { + private final GeckoMediaDrm.Callbacks mProxyCallbacks; + + CallbacksForwarder(final Callbacks callbacks) { + assertTrue(callbacks != null); + mProxyCallbacks = callbacks; + } + + @Override + public void onSessionCreated( + final int createSessionToken, + final int promiseId, + final byte[] sessionId, + final byte[] request) { + mProxyCallbacks.onSessionCreated(createSessionToken, promiseId, sessionId, request); + } + + @Override + public void onSessionUpdated(final int promiseId, final byte[] sessionId) { + mProxyCallbacks.onSessionUpdated(promiseId, sessionId); + } + + @Override + public void onSessionClosed(final int promiseId, final byte[] sessionId) { + mProxyCallbacks.onSessionClosed(promiseId, sessionId); + } + + @Override + public void onSessionMessage( + final byte[] sessionId, final int sessionMessageType, final byte[] request) { + mProxyCallbacks.onSessionMessage(sessionId, sessionMessageType, request); + } + + @Override + public void onSessionError(final byte[] sessionId, final String message) { + mProxyCallbacks.onSessionError(sessionId, message); + } + + @Override + public void onSessionBatchedKeyChanged( + final byte[] sessionId, final SessionKeyInfo[] keyInfos) { + mProxyCallbacks.onSessionBatchedKeyChanged(sessionId, keyInfos); + } + + @Override + public void onRejectPromise(final int promiseId, final String message) { + mProxyCallbacks.onRejectPromise(promiseId, message); + } + } // CallbacksForwarder + + /* package-private */ static void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + public RemoteMediaDrmBridge(final IMediaDrmBridge remoteBridge) { + assertTrue(remoteBridge != null); + mRemote = remoteBridge; + } + + @Override + public synchronized void setCallbacks(final Callbacks callbacks) { + if (DEBUG) Log.d(LOGTAG, "setCallbacks()"); + assertTrue(callbacks != null); + assertTrue(mRemote != null); + + mCallbacksFwd = new CallbacksForwarder(callbacks); + try { + mRemote.setCallbacks(mCallbacksFwd); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception during setCallbacks", e); + } + } + + @Override + public synchronized void createSession( + final int createSessionToken, + final int promiseId, + final String initDataType, + final byte[] initData) { + if (DEBUG) Log.d(LOGTAG, "createSession()"); + + try { + mRemote.createSession(createSessionToken, promiseId, initDataType, initData); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception while creating remote session.", e); + mCallbacksFwd.onRejectPromise(promiseId, "Failed to create session."); + } + } + + @Override + public synchronized void updateSession( + final int promiseId, final String sessionId, final byte[] response) { + if (DEBUG) Log.d(LOGTAG, "updateSession()"); + + try { + mRemote.updateSession(promiseId, sessionId, response); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception while updating remote session.", e); + mCallbacksFwd.onRejectPromise(promiseId, "Failed to update session."); + } + } + + @Override + public synchronized void closeSession(final int promiseId, final String sessionId) { + if (DEBUG) Log.d(LOGTAG, "closeSession()"); + + try { + mRemote.closeSession(promiseId, sessionId); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception while closing remote session.", e); + mCallbacksFwd.onRejectPromise(promiseId, "Failed to close session."); + } + } + + @Override + public synchronized void release() { + if (DEBUG) Log.d(LOGTAG, "release()"); + + try { + mRemote.release(); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception while releasing RemoteDrmBridge.", e); + } + RemoteManager.getInstance().onRemoteMediaDrmBridgeReleased(mRemote); + mRemote = null; + mCallbacksFwd = null; + } + + @Override + public synchronized MediaCrypto getMediaCrypto() { + if (DEBUG) Log.d(LOGTAG, "getMediaCrypto(), should not enter here!"); + assertTrue(false); + return null; + } + + @Override + public synchronized void setServerCertificate(final byte[] cert) { + try { + mRemote.setServerCertificate(cert); + } catch (final Exception e) { + Log.e(LOGTAG, "Got exception while setting server certificate.", e); + throw new RuntimeException(e); + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridgeStub.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridgeStub.java new file mode 100644 index 0000000000..8f9e42fde1 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/RemoteMediaDrmBridgeStub.java @@ -0,0 +1,248 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCrypto; +import android.os.Build; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; +import java.util.ArrayList; + +final class RemoteMediaDrmBridgeStub extends IMediaDrmBridge.Stub + implements IBinder.DeathRecipient { + private static final String LOGTAG = "RemoteDrmBridgeStub"; + private static final boolean DEBUG = false; + private volatile IMediaDrmBridgeCallbacks mCallbacks = null; + + // Underlying bridge implmenetaion, i.e. GeckoMediaDrmBrdigeV21. + private GeckoMediaDrm mBridge = null; + + // mStubId is initialized during stub construction. It should be a unique + // string which is generated in MediaDrmProxy in Fennec App process and is + // used for Codec to obtain corresponding MediaCrypto as input to achieve + // decryption. + // The generated stubId will be delivered to Codec via a code path starting + // from MediaDrmProxy -> MediaDrmCDMProxy -> RemoteDataDecoder => IPC => Codec. + private String mStubId = ""; + + public static final ArrayList<RemoteMediaDrmBridgeStub> mBridgeStubs = + new ArrayList<RemoteMediaDrmBridgeStub>(); + + private String getId() { + return mStubId; + } + + private MediaCrypto getMediaCryptoFromBridge() { + return mBridge != null ? mBridge.getMediaCrypto() : null; + } + + public static synchronized MediaCrypto getMediaCrypto(final String stubId) { + if (DEBUG) Log.d(LOGTAG, "getMediaCrypto()"); + + for (int i = 0; i < mBridgeStubs.size(); i++) { + if (mBridgeStubs.get(i) != null && mBridgeStubs.get(i).getId().equals(stubId)) { + return mBridgeStubs.get(i).getMediaCryptoFromBridge(); + } + } + return null; + } + + // Callback to RemoteMediaDrmBridge. + private final class Callbacks implements GeckoMediaDrm.Callbacks { + private IMediaDrmBridgeCallbacks mRemoteCallbacks; + + public Callbacks(final IMediaDrmBridgeCallbacks remote) { + mRemoteCallbacks = remote; + } + + @Override + public void onSessionCreated( + final int createSessionToken, + final int promiseId, + final byte[] sessionId, + final byte[] request) { + if (DEBUG) Log.d(LOGTAG, "onSessionCreated()"); + try { + mRemoteCallbacks.onSessionCreated(createSessionToken, promiseId, sessionId, request); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onSessionUpdated(final int promiseId, final byte[] sessionId) { + if (DEBUG) Log.d(LOGTAG, "onSessionUpdated()"); + try { + mRemoteCallbacks.onSessionUpdated(promiseId, sessionId); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onSessionClosed(final int promiseId, final byte[] sessionId) { + if (DEBUG) Log.d(LOGTAG, "onSessionClosed()"); + try { + mRemoteCallbacks.onSessionClosed(promiseId, sessionId); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onSessionMessage( + final byte[] sessionId, final int sessionMessageType, final byte[] request) { + if (DEBUG) Log.d(LOGTAG, "onSessionMessage()"); + try { + mRemoteCallbacks.onSessionMessage(sessionId, sessionMessageType, request); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onSessionError(final byte[] sessionId, final String message) { + if (DEBUG) Log.d(LOGTAG, "onSessionError()"); + try { + mRemoteCallbacks.onSessionError(sessionId, message); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onSessionBatchedKeyChanged( + final byte[] sessionId, final SessionKeyInfo[] keyInfos) { + if (DEBUG) Log.d(LOGTAG, "onSessionBatchedKeyChanged()"); + try { + mRemoteCallbacks.onSessionBatchedKeyChanged(sessionId, keyInfos); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public void onRejectPromise(final int promiseId, final String message) { + if (DEBUG) Log.d(LOGTAG, "onRejectPromise()"); + try { + mRemoteCallbacks.onRejectPromise(promiseId, message); + } catch (final RemoteException e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + } + + /* package-private */ void assertTrue(final boolean condition) { + if (DEBUG && !condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + RemoteMediaDrmBridgeStub(final String keySystem, final String stubId) throws RemoteException { + try { + if (Build.VERSION.SDK_INT < 23) { + mBridge = new GeckoMediaDrmBridgeV21(keySystem); + } else { + mBridge = new GeckoMediaDrmBridgeV23(keySystem); + } + mStubId = stubId; + mBridgeStubs.add(this); + } catch (final Exception e) { + throw new RemoteException("RemoteMediaDrmBridgeStub cannot create bridge implementation."); + } + } + + @Override + public synchronized void setCallbacks(final IMediaDrmBridgeCallbacks callbacks) + throws RemoteException { + if (DEBUG) Log.d(LOGTAG, "setCallbacks()"); + assertTrue(mBridge != null); + assertTrue(callbacks != null); + mCallbacks = callbacks; + callbacks.asBinder().linkToDeath(this, 0); + mBridge.setCallbacks(new Callbacks(mCallbacks)); + } + + @Override + public synchronized void createSession( + final int createSessionToken, + final int promiseId, + final String initDataType, + final byte[] initData) + throws RemoteException { + if (DEBUG) Log.d(LOGTAG, "createSession()"); + try { + assertTrue(mCallbacks != null); + assertTrue(mBridge != null); + mBridge.createSession(createSessionToken, promiseId, initDataType, initData); + } catch (final Exception e) { + Log.e(LOGTAG, "Failed to createSession.", e); + mCallbacks.onRejectPromise(promiseId, "Failed to createSession."); + } + } + + @Override + public synchronized void updateSession( + final int promiseId, final String sessionId, final byte[] response) throws RemoteException { + if (DEBUG) Log.d(LOGTAG, "updateSession()"); + try { + assertTrue(mCallbacks != null); + assertTrue(mBridge != null); + mBridge.updateSession(promiseId, sessionId, response); + } catch (final Exception e) { + Log.e(LOGTAG, "Failed to updateSession.", e); + mCallbacks.onRejectPromise(promiseId, "Failed to updateSession."); + } + } + + @Override + public synchronized void closeSession(final int promiseId, final String sessionId) + throws RemoteException { + if (DEBUG) Log.d(LOGTAG, "closeSession()"); + try { + assertTrue(mCallbacks != null); + assertTrue(mBridge != null); + mBridge.closeSession(promiseId, sessionId); + } catch (final Exception e) { + Log.e(LOGTAG, "Failed to closeSession.", e); + mCallbacks.onRejectPromise(promiseId, "Failed to closeSession."); + } + } + + // IBinder.DeathRecipient + @Override + public synchronized void binderDied() { + Log.e(LOGTAG, "Binder died !!"); + try { + release(); + } catch (final Exception e) { + Log.e(LOGTAG, "Exception ! Dead recipient !!", e); + } + } + + @Override + public synchronized void release() { + if (DEBUG) Log.d(LOGTAG, "release()"); + mBridgeStubs.remove(this); + if (mBridge != null) { + mBridge.release(); + mBridge = null; + } + mCallbacks.asBinder().unlinkToDeath(this, 0); + mCallbacks = null; + mStubId = ""; + } + + @Override + public synchronized void setServerCertificate(final byte[] cert) { + try { + mBridge.setServerCertificate(cert); + } catch (final IllegalStateException e) { + Log.e(LOGTAG, "Failed to setServerCertificate.", e); + throw e; + } + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Sample.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Sample.java new file mode 100644 index 0000000000..baa6737427 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Sample.java @@ -0,0 +1,291 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.annotation.SuppressLint; +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodec.CryptoInfo; +import android.os.Build; +import android.os.Parcel; +import android.os.Parcelable; +import androidx.annotation.ChecksSdkIntAtLeast; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import org.mozilla.gecko.annotation.WrapForJNI; + +// Parcelable carrying input/output sample data and info cross process. +public final class Sample implements Parcelable { + public static final Sample EOS; + + static { + final BufferInfo eosInfo = new BufferInfo(); + EOS = new Sample(); + EOS.info.set(0, 0, Long.MIN_VALUE, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } + + @WrapForJNI public long session; + + public static final int NO_BUFFER = -1; + + public int bufferId = NO_BUFFER; + @WrapForJNI public BufferInfo info = new BufferInfo(); + public CryptoInfo cryptoInfo; + + // Simple Linked list for recycling objects. + // Used to nodify Sample objects. Do not marshal/unmarshal. + private Sample mNext; + private static Sample sPool = new Sample(); + private static int sPoolSize = 1; + + private Sample() {} + + private void readInfo(final Parcel in) { + final int offset = in.readInt(); + final int size = in.readInt(); + final long pts = in.readLong(); + final int flags = in.readInt(); + + info.set(offset, size, pts, flags); + } + + private void readCrypto(final Parcel in) { + final int hasCryptoInfo = in.readInt(); + if (hasCryptoInfo == 0) { + cryptoInfo = null; + return; + } + + final byte[] iv = in.createByteArray(); + final byte[] key = in.createByteArray(); + final int mode = in.readInt(); + final int[] numBytesOfClearData = in.createIntArray(); + final int[] numBytesOfEncryptedData = in.createIntArray(); + final int numSubSamples = in.readInt(); + + if (cryptoInfo == null) { + cryptoInfo = new CryptoInfo(); + } + cryptoInfo.set(numSubSamples, numBytesOfClearData, numBytesOfEncryptedData, key, iv, mode); + if (supportsCryptoPattern()) { + final int numEncryptBlocks = in.readInt(); + final int numSkipBlocks = in.readInt(); + cryptoInfo.setPattern(new CryptoInfo.Pattern(numEncryptBlocks, numSkipBlocks)); + } + } + + public Sample set(final BufferInfo info, final CryptoInfo cryptoInfo) { + setBufferInfo(info); + setCryptoInfo(cryptoInfo); + return this; + } + + public void setBufferInfo(final BufferInfo info) { + this.info.set(0, info.size, info.presentationTimeUs, info.flags); + } + + public void setCryptoInfo(final CryptoInfo crypto) { + if (crypto == null) { + cryptoInfo = null; + return; + } + + if (cryptoInfo == null) { + cryptoInfo = new CryptoInfo(); + } + cryptoInfo.set( + crypto.numSubSamples, + crypto.numBytesOfClearData, + crypto.numBytesOfEncryptedData, + crypto.key, + crypto.iv, + crypto.mode); + if (supportsCryptoPattern()) { + final CryptoInfo.Pattern pattern = getCryptoPatternCompat(crypto); + if (pattern == null) { + return; + } + cryptoInfo.setPattern(pattern); + } + } + + @WrapForJNI + public void dispose() { + if (isEOS()) { + return; + } + + bufferId = NO_BUFFER; + info.set(0, 0, 0, 0); + if (cryptoInfo != null) { + cryptoInfo.set(0, null, null, null, null, 0); + } + + // Recycle it. + synchronized (CREATOR) { + this.mNext = sPool; + sPool = this; + sPoolSize++; + } + } + + public boolean isEOS() { + return (this == EOS) || ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0); + } + + public static Sample obtain() { + synchronized (CREATOR) { + Sample s = null; + if (sPoolSize > 0) { + s = sPool; + sPool = s.mNext; + s.mNext = null; + sPoolSize--; + } else { + s = new Sample(); + } + return s; + } + } + + public static final Creator<Sample> CREATOR = + new Creator<Sample>() { + @Override + public Sample createFromParcel(final Parcel in) { + return obtainSample(in); + } + + @Override + public Sample[] newArray(final int size) { + return new Sample[size]; + } + + private Sample obtainSample(final Parcel in) { + final Sample s = obtain(); + s.session = in.readLong(); + s.bufferId = in.readInt(); + s.readInfo(in); + s.readCrypto(in); + return s; + } + }; + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(final Parcel dest, final int parcelableFlags) { + dest.writeLong(session); + dest.writeInt(bufferId); + writeInfo(dest); + writeCrypto(dest); + } + + private void writeInfo(final Parcel dest) { + dest.writeInt(info.offset); + dest.writeInt(info.size); + dest.writeLong(info.presentationTimeUs); + dest.writeInt(info.flags); + } + + private void writeCrypto(final Parcel dest) { + if (cryptoInfo != null) { + dest.writeInt(1); + dest.writeByteArray(cryptoInfo.iv); + dest.writeByteArray(cryptoInfo.key); + dest.writeInt(cryptoInfo.mode); + dest.writeIntArray(cryptoInfo.numBytesOfClearData); + dest.writeIntArray(cryptoInfo.numBytesOfEncryptedData); + dest.writeInt(cryptoInfo.numSubSamples); + if (supportsCryptoPattern()) { + final CryptoInfo.Pattern pattern = getCryptoPatternCompat(cryptoInfo); + if (pattern != null) { + dest.writeInt(pattern.getEncryptBlocks()); + dest.writeInt(pattern.getSkipBlocks()); + } else { + // Couldn't get pattern - write default values + dest.writeInt(0); + dest.writeInt(0); + } + } + } else { + dest.writeInt(0); + } + } + + public static byte[] byteArrayFromBuffer( + final ByteBuffer buffer, final int offset, final int size) { + if (buffer == null || buffer.capacity() == 0 || size == 0) { + return null; + } + if (buffer.hasArray() && offset == 0 && buffer.array().length == size) { + return buffer.array(); + } + final int length = Math.min(offset + size, buffer.capacity()) - offset; + final byte[] bytes = new byte[length]; + buffer.position(offset); + buffer.get(bytes); + return bytes; + } + + @Override + public String toString() { + if (isEOS()) { + return "EOS sample"; + } + + final StringBuilder str = new StringBuilder(); + str.append("{ session#:") + .append(session) + .append(", buffer#") + .append(bufferId) + .append(", info=") + .append("{ offset=") + .append(info.offset) + .append(", size=") + .append(info.size) + .append(", pts=") + .append(info.presentationTimeUs) + .append(", flags=") + .append(Integer.toHexString(info.flags)) + .append(" }") + .append(" }"); + return str.toString(); + } + + @ChecksSdkIntAtLeast(api = android.os.Build.VERSION_CODES.N) + public static boolean supportsCryptoPattern() { + return Build.VERSION.SDK_INT >= 24; + } + + @SuppressLint("DiscouragedPrivateApi") + public static CryptoInfo.Pattern getCryptoPatternCompat(final CryptoInfo cryptoInfo) { + if (!supportsCryptoPattern()) { + return null; + } + // getPattern() added in API 31: + // https://developer.android.com/reference/android/media/MediaCodec.CryptoInfo#getPattern() + if (Build.VERSION.SDK_INT >= 31) { + return cryptoInfo.getPattern(); + } + + // CryptoInfo.Pattern added in API 24: + // https://developer.android.com/reference/android/media/MediaCodec.CryptoInfo.Pattern + if (Build.VERSION.SDK_INT >= 24) { + try { + // Without getPattern(), no way to access the pattern without reflection. + // https://cs.android.com/android/platform/superproject/+/android-11.0.0_r1:frameworks/base/media/java/android/media/MediaCodec.java;l=2718;drc=3c715d5778e15dc84082e63dc65b382d31fe8e45 + final Field patternField = CryptoInfo.class.getDeclaredField("pattern"); + patternField.setAccessible(true); + return (CryptoInfo.Pattern) patternField.get(cryptoInfo); + } catch (final NoSuchFieldException | IllegalAccessException e) { + return null; + } + } + return null; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SampleBuffer.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SampleBuffer.java new file mode 100644 index 0000000000..e6b242708d --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SampleBuffer.java @@ -0,0 +1,101 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.os.Parcel; +import android.os.Parcelable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.mozilla.gecko.annotation.WrapForJNI; +import org.mozilla.gecko.mozglue.SharedMemory; + +public final class SampleBuffer implements Parcelable { + private SharedMemory mSharedMem; + + /* package */ + public SampleBuffer(final SharedMemory sharedMem) { + mSharedMem = sharedMem; + } + + protected SampleBuffer(final Parcel in) { + mSharedMem = in.readParcelable(SampleBuffer.class.getClassLoader()); + } + + @Override + public void writeToParcel(final Parcel dest, final int flags) { + dest.writeParcelable(mSharedMem, flags); + } + + @Override + public int describeContents() { + return 0; + } + + public static final Creator<SampleBuffer> CREATOR = + new Creator<SampleBuffer>() { + @Override + public SampleBuffer createFromParcel(final Parcel in) { + return new SampleBuffer(in); + } + + @Override + public SampleBuffer[] newArray(final int size) { + return new SampleBuffer[size]; + } + }; + + public int capacity() { + return mSharedMem != null ? mSharedMem.getSize() : 0; + } + + public void readFromByteBuffer(final ByteBuffer src, final int offset, final int size) + throws IOException { + if (!src.isDirect()) { + throw new IOException("SharedMemBuffer only support reading from direct byte buffer."); + } + try { + nativeReadFromDirectBuffer(src, mSharedMem.getPointer(), offset, size); + mSharedMem.flush(); + } catch (final NullPointerException e) { + throw new IOException(e); + } + } + + private static native void nativeReadFromDirectBuffer( + ByteBuffer src, long dest, int offset, int size); + + @WrapForJNI + public void writeToByteBuffer(final ByteBuffer dest, final int offset, final int size) + throws IOException { + if (!dest.isDirect()) { + throw new IOException("SharedMemBuffer only support writing to direct byte buffer."); + } + try { + nativeWriteToDirectBuffer(mSharedMem.getPointer(), dest, offset, size); + } catch (final NullPointerException e) { + throw new IOException(e); + } + } + + private static native void nativeWriteToDirectBuffer( + long src, ByteBuffer dest, int offset, int size); + + public void dispose() { + if (mSharedMem != null) { + mSharedMem.dispose(); + mSharedMem = null; + } + } + + @WrapForJNI + public boolean isValid() { + return mSharedMem != null; + } + + @Override + public String toString() { + return "Buffer: " + mSharedMem; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SamplePool.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SamplePool.java new file mode 100644 index 0000000000..a2101b3aeb --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SamplePool.java @@ -0,0 +1,154 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.media.MediaCodec; +import android.util.SparseArray; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.mozilla.gecko.mozglue.SharedMemory; + +final class SamplePool { + private static final class Impl { + private final String mName; + private int mDefaultBufferSize = 4096; + private final List<Sample> mRecycledSamples = new ArrayList<>(); + private final boolean mBufferless; + + private int mNextBufferId = Sample.NO_BUFFER + 1; + private SparseArray<SampleBuffer> mBuffers = new SparseArray<>(); + + private Impl(final String name, final boolean bufferless) { + mName = name; + mBufferless = bufferless; + } + + private void setDefaultBufferSize(final int size) { + if (mBufferless) { + throw new IllegalStateException("Setting buffer size of a bufferless pool is not allowed"); + } + mDefaultBufferSize = size; + } + + private synchronized Sample obtain(final int size) { + if (!mRecycledSamples.isEmpty()) { + return mRecycledSamples.remove(0); + } + + if (mBufferless) { + return Sample.obtain(); + } else { + return allocateSampleAndBuffer(size); + } + } + + private Sample allocateSampleAndBuffer(final int size) { + final int id = mNextBufferId++; + try { + final SharedMemory shm = new SharedMemory(id, Math.max(size, mDefaultBufferSize)); + mBuffers.put((Integer) id, new SampleBuffer(shm)); + final Sample s = Sample.obtain(); + s.bufferId = id; + return s; + } catch (final NoSuchMethodException | IOException e) { + mBuffers.delete(id); + throw new UnsupportedOperationException(e); + } + } + + private synchronized SampleBuffer getBuffer(final int id) { + return mBuffers.get(id); + } + + private synchronized void recycle(final Sample recycled) { + if (mBufferless || isUsefulSample(recycled)) { + mRecycledSamples.add(recycled); + } else { + disposeSample(recycled); + } + } + + private boolean isUsefulSample(final Sample sample) { + return mBuffers.get(sample.bufferId).capacity() >= mDefaultBufferSize; + } + + private synchronized void clear() { + for (final Sample s : mRecycledSamples) { + disposeSample(s); + } + mRecycledSamples.clear(); + + for (int i = 0; i < mBuffers.size(); ++i) { + mBuffers.valueAt(i).dispose(); + } + mBuffers.clear(); + } + + private void disposeSample(final Sample sample) { + if (sample.bufferId != Sample.NO_BUFFER) { + mBuffers.get(sample.bufferId).dispose(); + mBuffers.delete(sample.bufferId); + } + sample.dispose(); + } + + @Override + protected void finalize() { + clear(); + } + } + + private final Impl mInputs; + private final Impl mOutputs; + + /* package */ SamplePool(final String name, final boolean renderToSurface) { + mInputs = new Impl(name + " input sample pool", false); + // Buffers are useless when rendering to surface. + mOutputs = new Impl(name + " output sample pool", renderToSurface); + } + + /* package */ void setInputBufferSize(final int size) { + mInputs.setDefaultBufferSize(size); + } + + /* package */ void setOutputBufferSize(final int size) { + mOutputs.setDefaultBufferSize(size); + } + + /* package */ Sample obtainInput(final int size) { + final Sample input = mInputs.obtain(size); + input.info.set(0, 0, 0, 0); + return input; + } + + /* package */ Sample obtainOutput(final MediaCodec.BufferInfo info) { + final Sample output = mOutputs.obtain(info.size); + output.info.set(0, info.size, info.presentationTimeUs, info.flags); + return output; + } + + /* package */ void recycleInput(final Sample sample) { + sample.cryptoInfo = null; + mInputs.recycle(sample); + } + + /* package */ void recycleOutput(final Sample sample) { + mOutputs.recycle(sample); + } + + /* package */ void reset() { + mInputs.clear(); + mOutputs.clear(); + } + + /* package */ SampleBuffer getInputBuffer(final int id) { + return mInputs.getBuffer(id); + } + + /* package */ SampleBuffer getOutputBuffer(final int id) { + return mOutputs.getBuffer(id); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SessionKeyInfo.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SessionKeyInfo.java new file mode 100644 index 0000000000..5e70a6f2a7 --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/SessionKeyInfo.java @@ -0,0 +1,50 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.os.Parcel; +import android.os.Parcelable; +import org.mozilla.gecko.annotation.WrapForJNI; + +public final class SessionKeyInfo implements Parcelable { + @WrapForJNI public byte[] keyId; + + @WrapForJNI public int status; + + @WrapForJNI + public SessionKeyInfo(final byte[] keyId, final int status) { + this.keyId = keyId; + this.status = status; + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(final Parcel dest, final int parcelableFlags) { + dest.writeByteArray(keyId); + dest.writeInt(status); + } + + public static final Creator<SessionKeyInfo> CREATOR = + new Creator<SessionKeyInfo>() { + @Override + public SessionKeyInfo createFromParcel(final Parcel in) { + return new SessionKeyInfo(in); + } + + @Override + public SessionKeyInfo[] newArray(final int size) { + return new SessionKeyInfo[size]; + } + }; + + private SessionKeyInfo(final Parcel src) { + keyId = src.createByteArray(); + status = src.readInt(); + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Utils.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Utils.java new file mode 100644 index 0000000000..5cc32e127c --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Utils.java @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.media; + +import android.util.Log; + +public class Utils { + public static long getThreadId() { + final Thread t = Thread.currentThread(); + return t.getId(); + } + + public static String getThreadSignature() { + final Thread t = Thread.currentThread(); + final long l = t.getId(); + final String name = t.getName(); + final long p = t.getPriority(); + final String gname = t.getThreadGroup().getName(); + return (name + ":(id)" + l + ":(priority)" + p + ":(group)" + gname); + } + + public static void logThreadSignature() { + Log.d("ThreadUtils", getThreadSignature()); + } + + private static final char[] hexArray = "0123456789ABCDEF".toCharArray(); + + public static String bytesToHex(final byte[] bytes) { + final char[] hexChars = new char[bytes.length * 2]; + for (int j = 0; j < bytes.length; j++) { + final int v = bytes[j] & 0xFF; + hexChars[j * 2] = hexArray[v >>> 4]; + hexChars[j * 2 + 1] = hexArray[v & 0x0F]; + } + return new String(hexChars); + } +} |