diff options
Diffstat (limited to 'mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical')
6 files changed, 783 insertions, 0 deletions
diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java new file mode 100644 index 0000000000..87bd94c5bc --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +/** Listens camera motion. */ +public interface CameraMotionListener { + + /** + * Called when a new camera motion is read. This method is called on the playback thread. + * + * @param timeUs The presentation time of the data. + * @param rotation Angle axis orientation in radians representing the rotation from camera + * coordinate system to world coordinate system. + */ + void onCameraMotion(long timeUs, float[] rotation); + + /** Called when the camera motion track position is reset or the track is disabled. */ + void onCameraMotionReset(); +} diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java new file mode 100644 index 0000000000..378363aca0 --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +import androidx.annotation.Nullable; +import org.mozilla.thirdparty.com.google.android.exoplayer2.BaseRenderer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.ExoPlaybackException; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Format; +import org.mozilla.thirdparty.com.google.android.exoplayer2.FormatHolder; +import org.mozilla.thirdparty.com.google.android.exoplayer2.Renderer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.RendererCapabilities; +import org.mozilla.thirdparty.com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.MimeTypes; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.ParsableByteArray; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Util; +import java.nio.ByteBuffer; + +/** A {@link Renderer} that parses the camera motion track. */ +public class CameraMotionRenderer extends BaseRenderer { + + // The amount of time to read samples ahead of the current time. + private static final int SAMPLE_WINDOW_DURATION_US = 100000; + + private final DecoderInputBuffer buffer; + private final ParsableByteArray scratch; + + private long offsetUs; + @Nullable private CameraMotionListener listener; + private long lastTimestampUs; + + public CameraMotionRenderer() { + super(C.TRACK_TYPE_CAMERA_MOTION); + buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + scratch = new ParsableByteArray(); + } + + @Override + @Capabilities + public int supportsFormat(Format format) { + return MimeTypes.APPLICATION_CAMERA_MOTION.equals(format.sampleMimeType) + ? RendererCapabilities.create(FORMAT_HANDLED) + : RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + } + + @Override + public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { + if (messageType == C.MSG_SET_CAMERA_MOTION_LISTENER) { + listener = (CameraMotionListener) message; + } else { + super.handleMessage(messageType, message); + } + } + + @Override + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { + this.offsetUs = offsetUs; + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + resetListener(); + } + + @Override + protected void onDisabled() { + resetListener(); + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + // Keep reading available samples as long as the sample time is not too far into the future. + while (!hasReadStreamToEnd() && lastTimestampUs < positionUs + SAMPLE_WINDOW_DURATION_US) { + buffer.clear(); + FormatHolder formatHolder = getFormatHolder(); + int result = readSource(formatHolder, buffer, /* formatRequired= */ false); + if (result != C.RESULT_BUFFER_READ || buffer.isEndOfStream()) { + return; + } + + buffer.flip(); + lastTimestampUs = buffer.timeUs; + if (listener != null) { + float[] rotation = parseMetadata(Util.castNonNull(buffer.data)); + if (rotation != null) { + Util.castNonNull(listener).onCameraMotion(lastTimestampUs - offsetUs, rotation); + } + } + } + } + + @Override + public boolean isEnded() { + return hasReadStreamToEnd(); + } + + @Override + public boolean isReady() { + return true; + } + + private @Nullable float[] parseMetadata(ByteBuffer data) { + if (data.remaining() != 16) { + return null; + } + scratch.reset(data.array(), data.limit()); + scratch.setPosition(data.arrayOffset() + 4); // skip reserved bytes too. + float[] result = new float[3]; + for (int i = 0; i < 3; i++) { + result[i] = Float.intBitsToFloat(scratch.readLittleEndianInt()); + } + return result; + } + + private void resetListener() { + lastTimestampUs = 0; + if (listener != null) { + listener.onCameraMotionReset(); + } + } +} diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java new file mode 100644 index 0000000000..450058fb6a --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +import android.opengl.Matrix; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.TimedValueQueue; + +/** + * This class serves multiple purposes: + * + * <ul> + * <li>Queues the rotation metadata extracted from camera motion track. + * <li>Converts the metadata to rotation matrices in OpenGl coordinate system. + * <li>Recenters the rotations to componsate the yaw of the initial rotation. + * </ul> + */ +public final class FrameRotationQueue { + private final float[] recenterMatrix; + private final float[] rotationMatrix; + private final TimedValueQueue<float[]> rotations; + private boolean recenterMatrixComputed; + + public FrameRotationQueue() { + recenterMatrix = new float[16]; + rotationMatrix = new float[16]; + rotations = new TimedValueQueue<>(); + } + + /** + * Sets a rotation for a given timestamp. + * + * @param timestampUs Timestamp of the rotation. + * @param angleAxis Angle axis orientation in radians representing the rotation from camera + * coordinate system to world coordinate system. + */ + public void setRotation(long timestampUs, float[] angleAxis) { + rotations.add(timestampUs, angleAxis); + } + + /** Removes all of the rotations and forces rotations to be recentered. */ + public void reset() { + rotations.clear(); + recenterMatrixComputed = false; + } + + /** + * Copies the rotation matrix with the greatest timestamp which is less than or equal to the given + * timestamp to {@code matrix}. Removes all older rotations and the returned one from the queue. + * Does nothing if there is no such rotation. + * + * @param matrix The rotation matrix. + * @param timestampUs The time in microseconds to query the rotation. + * @return Whether a rotation matrix is copied to {@code matrix}. + */ + public boolean pollRotationMatrix(float[] matrix, long timestampUs) { + float[] rotation = rotations.pollFloor(timestampUs); + if (rotation == null) { + return false; + } + // TODO [Internal: b/113315546]: Slerp between the floor and ceil rotation. + getRotationMatrixFromAngleAxis(rotationMatrix, rotation); + if (!recenterMatrixComputed) { + computeRecenterMatrix(recenterMatrix, rotationMatrix); + recenterMatrixComputed = true; + } + Matrix.multiplyMM(matrix, 0, recenterMatrix, 0, rotationMatrix, 0); + return true; + } + + /** + * Computes a recentering matrix from the given angle-axis rotation only accounting for yaw. Roll + * and tilt will not be compensated. + * + * @param recenterMatrix The recenter matrix. + * @param rotationMatrix The rotation matrix. + */ + public static void computeRecenterMatrix(float[] recenterMatrix, float[] rotationMatrix) { + // The re-centering matrix is computed as follows: + // recenter.row(2) = temp.col(2).transpose(); + // recenter.row(0) = recenter.row(1).cross(recenter.row(2)).normalized(); + // recenter.row(2) = recenter.row(0).cross(recenter.row(1)).normalized(); + // | temp[10] 0 -temp[8] 0| + // | 0 1 0 0| + // recenter = | temp[8] 0 temp[10] 0| + // | 0 0 0 1| + Matrix.setIdentityM(recenterMatrix, 0); + float normRowSqr = + rotationMatrix[10] * rotationMatrix[10] + rotationMatrix[8] * rotationMatrix[8]; + float normRow = (float) Math.sqrt(normRowSqr); + recenterMatrix[0] = rotationMatrix[10] / normRow; + recenterMatrix[2] = rotationMatrix[8] / normRow; + recenterMatrix[8] = -rotationMatrix[8] / normRow; + recenterMatrix[10] = rotationMatrix[10] / normRow; + } + + private static void getRotationMatrixFromAngleAxis(float[] matrix, float[] angleAxis) { + // Convert coordinates to OpenGL coordinates. + // CAMM motion metadata: +x right, +y down, and +z forward. + // OpenGL: +x right, +y up, -z forwards + float x = angleAxis[0]; + float y = -angleAxis[1]; + float z = -angleAxis[2]; + float angleRad = Matrix.length(x, y, z); + if (angleRad != 0) { + float angleDeg = (float) Math.toDegrees(angleRad); + Matrix.setRotateM(matrix, 0, angleDeg, x / angleRad, y / angleRad, z / angleRad); + } else { + Matrix.setIdentityM(matrix, 0); + } + } +} diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/Projection.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/Projection.java new file mode 100644 index 0000000000..e3d614cab3 --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/Projection.java @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +import androidx.annotation.IntDef; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C.StereoMode; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Assertions; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** The projection mesh used with 360/VR videos. */ +public final class Projection { + + /** Enforces allowed (sub) mesh draw modes. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @IntDef({DRAW_MODE_TRIANGLES, DRAW_MODE_TRIANGLES_STRIP, DRAW_MODE_TRIANGLES_FAN}) + public @interface DrawMode {} + /** Triangle draw mode. */ + public static final int DRAW_MODE_TRIANGLES = 0; + /** Triangle strip draw mode. */ + public static final int DRAW_MODE_TRIANGLES_STRIP = 1; + /** Triangle fan draw mode. */ + public static final int DRAW_MODE_TRIANGLES_FAN = 2; + + /** Number of position coordinates per vertex. */ + public static final int TEXTURE_COORDS_PER_VERTEX = 2; + /** Number of texture coordinates per vertex. */ + public static final int POSITION_COORDS_PER_VERTEX = 3; + + /** + * Generates a complete sphere equirectangular projection. + * + * @param stereoMode A {@link C.StereoMode} value. + */ + public static Projection createEquirectangular(@C.StereoMode int stereoMode) { + return createEquirectangular( + /* radius= */ 50, // Should be large enough that there are no stereo artifacts. + /* latitudes= */ 36, // Should be large enough to prevent videos looking wavy. + /* longitudes= */ 72, // Should be large enough to prevent videos looking wavy. + /* verticalFovDegrees= */ 180, + /* horizontalFovDegrees= */ 360, + stereoMode); + } + + /** + * Generates an equirectangular projection. + * + * @param radius Size of the sphere. Must be > 0. + * @param latitudes Number of rows that make up the sphere. Must be >= 1. + * @param longitudes Number of columns that make up the sphere. Must be >= 1. + * @param verticalFovDegrees Total latitudinal degrees that are covered by the sphere. Must be in + * (0, 180]. + * @param horizontalFovDegrees Total longitudinal degrees that are covered by the sphere.Must be + * in (0, 360]. + * @param stereoMode A {@link C.StereoMode} value. + * @return an equirectangular projection. + */ + public static Projection createEquirectangular( + float radius, + int latitudes, + int longitudes, + float verticalFovDegrees, + float horizontalFovDegrees, + @C.StereoMode int stereoMode) { + Assertions.checkArgument(radius > 0); + Assertions.checkArgument(latitudes >= 1); + Assertions.checkArgument(longitudes >= 1); + Assertions.checkArgument(verticalFovDegrees > 0 && verticalFovDegrees <= 180); + Assertions.checkArgument(horizontalFovDegrees > 0 && horizontalFovDegrees <= 360); + + // Compute angular size in radians of each UV quad. + float verticalFovRads = (float) Math.toRadians(verticalFovDegrees); + float horizontalFovRads = (float) Math.toRadians(horizontalFovDegrees); + float quadHeightRads = verticalFovRads / latitudes; + float quadWidthRads = horizontalFovRads / longitudes; + + // Each latitude strip has 2 * (longitudes quads + extra edge) vertices + 2 degenerate vertices. + int vertexCount = (2 * (longitudes + 1) + 2) * latitudes; + // Buffer to return. + float[] vertexData = new float[vertexCount * POSITION_COORDS_PER_VERTEX]; + float[] textureData = new float[vertexCount * TEXTURE_COORDS_PER_VERTEX]; + + // Generate the data for the sphere which is a set of triangle strips representing each + // latitude band. + int vOffset = 0; // Offset into the vertexData array. + int tOffset = 0; // Offset into the textureData array. + // (i, j) represents a quad in the equirectangular sphere. + for (int j = 0; j < latitudes; ++j) { // For each horizontal triangle strip. + // Each latitude band lies between the two phi values. Each vertical edge on a band lies on + // a theta value. + float phiLow = quadHeightRads * j - verticalFovRads / 2; + float phiHigh = quadHeightRads * (j + 1) - verticalFovRads / 2; + + for (int i = 0; i < longitudes + 1; ++i) { // For each vertical edge in the band. + for (int k = 0; k < 2; ++k) { // For low and high points on an edge. + // For each point, determine it's position in polar coordinates. + float phi = k == 0 ? phiLow : phiHigh; + float theta = quadWidthRads * i + (float) Math.PI - horizontalFovRads / 2; + + // Set vertex position data as Cartesian coordinates. + vertexData[vOffset++] = -(float) (radius * Math.sin(theta) * Math.cos(phi)); + vertexData[vOffset++] = (float) (radius * Math.sin(phi)); + vertexData[vOffset++] = (float) (radius * Math.cos(theta) * Math.cos(phi)); + + textureData[tOffset++] = i * quadWidthRads / horizontalFovRads; + textureData[tOffset++] = (j + k) * quadHeightRads / verticalFovRads; + + // Break up the triangle strip with degenerate vertices by copying first and last points. + if ((i == 0 && k == 0) || (i == longitudes && k == 1)) { + System.arraycopy( + vertexData, + vOffset - POSITION_COORDS_PER_VERTEX, + vertexData, + vOffset, + POSITION_COORDS_PER_VERTEX); + vOffset += POSITION_COORDS_PER_VERTEX; + System.arraycopy( + textureData, + tOffset - TEXTURE_COORDS_PER_VERTEX, + textureData, + tOffset, + TEXTURE_COORDS_PER_VERTEX); + tOffset += TEXTURE_COORDS_PER_VERTEX; + } + } + // Move on to the next vertical edge in the triangle strip. + } + // Move on to the next triangle strip. + } + SubMesh subMesh = + new SubMesh(SubMesh.VIDEO_TEXTURE_ID, vertexData, textureData, DRAW_MODE_TRIANGLES_STRIP); + return new Projection(new Mesh(subMesh), stereoMode); + } + + /** The Mesh corresponding to the left eye. */ + public final Mesh leftMesh; + /** + * The Mesh corresponding to the right eye. If {@code singleMesh} is true then this mesh is + * identical to {@link #leftMesh}. + */ + public final Mesh rightMesh; + /** The stereo mode. */ + public final @StereoMode int stereoMode; + /** Whether the left and right mesh are identical. */ + public final boolean singleMesh; + + /** + * Creates a Projection with single mesh. + * + * @param mesh the Mesh for both eyes. + * @param stereoMode A {@link StereoMode} value. + */ + public Projection(Mesh mesh, int stereoMode) { + this(mesh, mesh, stereoMode); + } + + /** + * Creates a Projection with dual mesh. Use {@link #Projection(Mesh, int)} if there is single mesh + * for both eyes. + * + * @param leftMesh the Mesh corresponding to the left eye. + * @param rightMesh the Mesh corresponding to the right eye. + * @param stereoMode A {@link C.StereoMode} value. + */ + public Projection(Mesh leftMesh, Mesh rightMesh, int stereoMode) { + this.leftMesh = leftMesh; + this.rightMesh = rightMesh; + this.stereoMode = stereoMode; + this.singleMesh = leftMesh == rightMesh; + } + + /** The sub mesh associated with the {@link Mesh}. */ + public static final class SubMesh { + /** Texture ID for video frames. */ + public static final int VIDEO_TEXTURE_ID = 0; + + /** Texture ID. */ + public final int textureId; + /** The drawing mode. One of {@link DrawMode}. */ + public final @DrawMode int mode; + /** The SubMesh vertices. */ + public final float[] vertices; + /** The SubMesh texture coordinates. */ + public final float[] textureCoords; + + public SubMesh(int textureId, float[] vertices, float[] textureCoords, @DrawMode int mode) { + this.textureId = textureId; + Assertions.checkArgument( + vertices.length * (long) TEXTURE_COORDS_PER_VERTEX + == textureCoords.length * (long) POSITION_COORDS_PER_VERTEX); + this.vertices = vertices; + this.textureCoords = textureCoords; + this.mode = mode; + } + + /** Returns the SubMesh vertex count. */ + public int getVertexCount() { + return vertices.length / POSITION_COORDS_PER_VERTEX; + } + } + + /** A Mesh associated with the projection scene. */ + public static final class Mesh { + private final SubMesh[] subMeshes; + + public Mesh(SubMesh... subMeshes) { + this.subMeshes = subMeshes; + } + + /** Returns the number of sub meshes. */ + public int getSubMeshCount() { + return subMeshes.length; + } + + /** Returns the SubMesh for the given index. */ + public SubMesh getSubMesh(int index) { + return subMeshes[index]; + } + } +} diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/ProjectionDecoder.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/ProjectionDecoder.java new file mode 100644 index 0000000000..cff4b2845d --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/ProjectionDecoder.java @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +import androidx.annotation.Nullable; +import org.mozilla.thirdparty.com.google.android.exoplayer2.C; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.ParsableBitArray; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.ParsableByteArray; +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.Util; +import org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical.Projection.Mesh; +import org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical.Projection.SubMesh; +import java.util.ArrayList; +import java.util.zip.Inflater; + +/** + * A decoder for the projection mesh. + * + * <p>The mesh boxes parsed are described at <a + * href="https://github.com/google/spatial-media/blob/master/docs/spherical-video-v2-rfc.md"> + * Spherical Video V2 RFC</a>. + * + * <p>The decoder does not perform CRC checks at the moment. + */ +public final class ProjectionDecoder { + + private static final int TYPE_YTMP = 0x79746d70; + private static final int TYPE_MSHP = 0x6d736870; + private static final int TYPE_RAW = 0x72617720; + private static final int TYPE_DFL8 = 0x64666c38; + private static final int TYPE_MESH = 0x6d657368; + private static final int TYPE_PROJ = 0x70726f6a; + + // Sanity limits to prevent a bad file from creating an OOM situation. We don't expect a mesh to + // exceed these limits. + private static final int MAX_COORDINATE_COUNT = 10000; + private static final int MAX_VERTEX_COUNT = 32 * 1000; + private static final int MAX_TRIANGLE_INDICES = 128 * 1000; + + private ProjectionDecoder() {} + + /* + * Decodes the projection data. + * + * @param projectionData The projection data. + * @param stereoMode A {@link C.StereoMode} value. + * @return The projection or null if the data can't be decoded. + */ + public static @Nullable Projection decode(byte[] projectionData, @C.StereoMode int stereoMode) { + ParsableByteArray input = new ParsableByteArray(projectionData); + // MP4 containers include the proj box but webm containers do not. + // Both containers use mshp. + ArrayList<Mesh> meshes = null; + try { + meshes = isProj(input) ? parseProj(input) : parseMshp(input); + } catch (ArrayIndexOutOfBoundsException ignored) { + // Do nothing. + } + if (meshes == null) { + return null; + } else { + switch (meshes.size()) { + case 1: + return new Projection(meshes.get(0), stereoMode); + case 2: + return new Projection(meshes.get(0), meshes.get(1), stereoMode); + case 0: + default: + return null; + } + } + } + + /** Returns true if the input contains a proj box. Indicates MP4 container. */ + private static boolean isProj(ParsableByteArray input) { + input.skipBytes(4); // size + int type = input.readInt(); + input.setPosition(0); + return type == TYPE_PROJ; + } + + private static @Nullable ArrayList<Mesh> parseProj(ParsableByteArray input) { + input.skipBytes(8); // size and type. + int position = input.getPosition(); + int limit = input.limit(); + while (position < limit) { + int childEnd = position + input.readInt(); + if (childEnd <= position || childEnd > limit) { + return null; + } + int childAtomType = input.readInt(); + // Some early files named the atom ytmp rather than mshp. + if (childAtomType == TYPE_YTMP || childAtomType == TYPE_MSHP) { + input.setLimit(childEnd); + return parseMshp(input); + } + position = childEnd; + input.setPosition(position); + } + return null; + } + + private static @Nullable ArrayList<Mesh> parseMshp(ParsableByteArray input) { + int version = input.readUnsignedByte(); + if (version != 0) { + return null; + } + input.skipBytes(7); // flags + crc. + int encoding = input.readInt(); + if (encoding == TYPE_DFL8) { + ParsableByteArray output = new ParsableByteArray(); + Inflater inflater = new Inflater(true); + try { + if (!Util.inflate(input, output, inflater)) { + return null; + } + } finally { + inflater.end(); + } + input = output; + } else if (encoding != TYPE_RAW) { + return null; + } + return parseRawMshpData(input); + } + + /** Parses MSHP data after the encoding_four_cc field. */ + private static @Nullable ArrayList<Mesh> parseRawMshpData(ParsableByteArray input) { + ArrayList<Mesh> meshes = new ArrayList<>(); + int position = input.getPosition(); + int limit = input.limit(); + while (position < limit) { + int childEnd = position + input.readInt(); + if (childEnd <= position || childEnd > limit) { + return null; + } + int childAtomType = input.readInt(); + if (childAtomType == TYPE_MESH) { + Mesh mesh = parseMesh(input); + if (mesh == null) { + return null; + } + meshes.add(mesh); + } + position = childEnd; + input.setPosition(position); + } + return meshes; + } + + private static @Nullable Mesh parseMesh(ParsableByteArray input) { + // Read the coordinates. + int coordinateCount = input.readInt(); + if (coordinateCount > MAX_COORDINATE_COUNT) { + return null; + } + float[] coordinates = new float[coordinateCount]; + for (int coordinate = 0; coordinate < coordinateCount; coordinate++) { + coordinates[coordinate] = input.readFloat(); + } + // Read the vertices. + int vertexCount = input.readInt(); + if (vertexCount > MAX_VERTEX_COUNT) { + return null; + } + + final double log2 = Math.log(2.0); + int coordinateCountSizeBits = (int) Math.ceil(Math.log(2.0 * coordinateCount) / log2); + + ParsableBitArray bitInput = new ParsableBitArray(input.data); + bitInput.setPosition(input.getPosition() * 8); + float[] vertices = new float[vertexCount * 5]; + int[] coordinateIndices = new int[5]; + int vertexIndex = 0; + for (int vertex = 0; vertex < vertexCount; vertex++) { + for (int i = 0; i < 5; i++) { + int coordinateIndex = + coordinateIndices[i] + decodeZigZag(bitInput.readBits(coordinateCountSizeBits)); + if (coordinateIndex >= coordinateCount || coordinateIndex < 0) { + return null; + } + vertices[vertexIndex++] = coordinates[coordinateIndex]; + coordinateIndices[i] = coordinateIndex; + } + } + + // Pad to next byte boundary + bitInput.setPosition(((bitInput.getPosition() + 7) & ~7)); + + int subMeshCount = bitInput.readBits(32); + SubMesh[] subMeshes = new SubMesh[subMeshCount]; + for (int i = 0; i < subMeshCount; i++) { + int textureId = bitInput.readBits(8); + int drawMode = bitInput.readBits(8); + int triangleIndexCount = bitInput.readBits(32); + if (triangleIndexCount > MAX_TRIANGLE_INDICES) { + return null; + } + int vertexCountSizeBits = (int) Math.ceil(Math.log(2.0 * vertexCount) / log2); + int index = 0; + float[] triangleVertices = new float[triangleIndexCount * 3]; + float[] textureCoords = new float[triangleIndexCount * 2]; + for (int counter = 0; counter < triangleIndexCount; counter++) { + index += decodeZigZag(bitInput.readBits(vertexCountSizeBits)); + if (index < 0 || index >= vertexCount) { + return null; + } + triangleVertices[counter * 3] = vertices[index * 5]; + triangleVertices[counter * 3 + 1] = vertices[index * 5 + 1]; + triangleVertices[counter * 3 + 2] = vertices[index * 5 + 2]; + textureCoords[counter * 2] = vertices[index * 5 + 3]; + textureCoords[counter * 2 + 1] = vertices[index * 5 + 4]; + } + subMeshes[i] = new SubMesh(textureId, triangleVertices, textureCoords, drawMode); + } + return new Mesh(subMeshes); + } + + /** + * Decodes Zigzag encoding as described in + * https://developers.google.com/protocol-buffers/docs/encoding#signed-integers + */ + private static int decodeZigZag(int n) { + return (n >> 1) ^ -(n & 1); + } +} diff --git a/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/package-info.java b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/package-info.java new file mode 100644 index 0000000000..7ab7fced0b --- /dev/null +++ b/mobile/android/exoplayer2/src/main/java/org/mozilla/thirdparty/com/google/android/exoplayer2/video/spherical/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package org.mozilla.thirdparty.com.google.android.exoplayer2.video.spherical; + +import org.mozilla.thirdparty.com.google.android.exoplayer2.util.NonNullApi; |