diff options
Diffstat (limited to 'third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java')
| -rw-r--r-- | third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java | 252 |
1 files changed, 252 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java new file mode 100644 index 0000000000..c855d4be41 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java @@ -0,0 +1,252 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.opengl.GLES20; +import android.opengl.GLException; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import org.webrtc.VideoFrame.I420Buffer; +import org.webrtc.VideoFrame.TextureBuffer; + +/** + * Class for converting OES textures to a YUV ByteBuffer. It can be constructed on any thread, but + * should only be operated from a single thread with an active EGL context. + */ +public final class YuvConverter { + private static final String TAG = "YuvConverter"; + + private static final String FRAGMENT_SHADER = + // Difference in texture coordinate corresponding to one + // sub-pixel in the x direction. + "uniform vec2 xUnit;\n" + // Color conversion coefficients, including constant term + + "uniform vec4 coeffs;\n" + + "\n" + + "void main() {\n" + // Since the alpha read from the texture is always 1, this could + // be written as a mat4 x vec4 multiply. However, that seems to + // give a worse framerate, possibly because the additional + // multiplies by 1.0 consume resources. + + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc - 1.5 * xUnit).rgb);\n" + + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc - 0.5 * xUnit).rgb);\n" + + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc + 0.5 * xUnit).rgb);\n" + + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc + 1.5 * xUnit).rgb);\n" + + "}\n"; + + private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks { + // Y'UV444 to RGB888, see https://en.wikipedia.org/wiki/YUV#Y%E2%80%B2UV444_to_RGB888_conversion + // We use the ITU-R BT.601 coefficients for Y, U and V. + // The values in Wikipedia are inaccurate, the accurate values derived from the spec are: + // Y = 0.299 * R + 0.587 * G + 0.114 * B + // U = -0.168736 * R - 0.331264 * G + 0.5 * B + 0.5 + // V = 0.5 * R - 0.418688 * G - 0.0813124 * B + 0.5 + // To map the Y-values to range [16-235] and U- and V-values to range [16-240], the matrix has + // been multiplied with matrix: + // {{219 / 255, 0, 0, 16 / 255}, + // {0, 224 / 255, 0, 16 / 255}, + // {0, 0, 224 / 255, 16 / 255}, + // {0, 0, 0, 1}} + private static final float[] yCoeffs = + new float[] {0.256788f, 0.504129f, 0.0979059f, 0.0627451f}; + private static final float[] uCoeffs = + new float[] {-0.148223f, -0.290993f, 0.439216f, 0.501961f}; + private static final float[] vCoeffs = + new float[] {0.439216f, -0.367788f, -0.0714274f, 0.501961f}; + + private int xUnitLoc; + private int coeffsLoc; + + private float[] coeffs; + private float stepSize; + + public void setPlaneY() { + coeffs = yCoeffs; + stepSize = 1.0f; + } + + public void setPlaneU() { + coeffs = uCoeffs; + stepSize = 2.0f; + } + + public void setPlaneV() { + coeffs = vCoeffs; + stepSize = 2.0f; + } + + @Override + public void onNewShader(GlShader shader) { + xUnitLoc = shader.getUniformLocation("xUnit"); + coeffsLoc = shader.getUniformLocation("coeffs"); + } + + @Override + public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, + int viewportWidth, int viewportHeight) { + GLES20.glUniform4fv(coeffsLoc, /* count= */ 1, coeffs, /* offset= */ 0); + // Matrix * (1;0;0;0) / (width / stepSize). Note that OpenGL uses column major order. + GLES20.glUniform2f( + xUnitLoc, stepSize * texMatrix[0] / frameWidth, stepSize * texMatrix[1] / frameWidth); + } + } + + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + private final GlTextureFrameBuffer i420TextureFrameBuffer = + new GlTextureFrameBuffer(GLES20.GL_RGBA); + private final ShaderCallbacks shaderCallbacks = new ShaderCallbacks(); + private final GlGenericDrawer drawer = new GlGenericDrawer(FRAGMENT_SHADER, shaderCallbacks); + private final VideoFrameDrawer videoFrameDrawer; + + /** + * This class should be constructed on a thread that has an active EGL context. + */ + public YuvConverter() { + this(new VideoFrameDrawer()); + } + + public YuvConverter(VideoFrameDrawer videoFrameDrawer) { + this.videoFrameDrawer = videoFrameDrawer; + threadChecker.detachThread(); + } + + /** Converts the texture buffer to I420. */ + @Nullable + public I420Buffer convert(TextureBuffer inputTextureBuffer) { + try { + return convertInternal(inputTextureBuffer); + } catch (GLException e) { + Logging.w(TAG, "Failed to convert TextureBuffer", e); + } + return null; + } + + private I420Buffer convertInternal(TextureBuffer inputTextureBuffer) { + TextureBuffer preparedBuffer = (TextureBuffer) videoFrameDrawer.prepareBufferForViewportSize( + inputTextureBuffer, inputTextureBuffer.getWidth(), inputTextureBuffer.getHeight()); + + // We draw into a buffer laid out like + // + // +---------+ + // | | + // | Y | + // | | + // | | + // +----+----+ + // | U | V | + // | | | + // +----+----+ + // + // In memory, we use the same stride for all of Y, U and V. The + // U data starts at offset `height` * `stride` from the Y data, + // and the V data starts at at offset |stride/2| from the U + // data, with rows of U and V data alternating. + // + // Now, it would have made sense to allocate a pixel buffer with + // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, + // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be + // unsupported by devices. So do the following hack: Allocate an + // RGBA buffer, of width `stride`/4. To render each of these + // large pixels, sample the texture at 4 different x coordinates + // and store the results in the four components. + // + // Since the V data needs to start on a boundary of such a + // larger pixel, it is not sufficient that `stride` is even, it + // has to be a multiple of 8 pixels. + final int frameWidth = preparedBuffer.getWidth(); + final int frameHeight = preparedBuffer.getHeight(); + final int stride = ((frameWidth + 7) / 8) * 8; + final int uvHeight = (frameHeight + 1) / 2; + // Total height of the combined memory layout. + final int totalHeight = frameHeight + uvHeight; + final ByteBuffer i420ByteBuffer = JniCommon.nativeAllocateByteBuffer(stride * totalHeight); + // Viewport width is divided by four since we are squeezing in four color bytes in each RGBA + // pixel. + final int viewportWidth = stride / 4; + + // Produce a frame buffer starting at top-left corner, not bottom-left. + final Matrix renderMatrix = new Matrix(); + renderMatrix.preTranslate(0.5f, 0.5f); + renderMatrix.preScale(1f, -1f); + renderMatrix.preTranslate(-0.5f, -0.5f); + + i420TextureFrameBuffer.setSize(viewportWidth, totalHeight); + + // Bind our framebuffer. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, i420TextureFrameBuffer.getFrameBufferId()); + GlUtil.checkNoGLES2Error("glBindFramebuffer"); + + // Draw Y. + shaderCallbacks.setPlaneY(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ 0, /* viewportY= */ 0, viewportWidth, + /* viewportHeight= */ frameHeight); + + // Draw U. + shaderCallbacks.setPlaneU(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ 0, /* viewportY= */ frameHeight, viewportWidth / 2, + /* viewportHeight= */ uvHeight); + + // Draw V. + shaderCallbacks.setPlaneV(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ viewportWidth / 2, /* viewportY= */ frameHeight, viewportWidth / 2, + /* viewportHeight= */ uvHeight); + + GLES20.glReadPixels(0, 0, i420TextureFrameBuffer.getWidth(), i420TextureFrameBuffer.getHeight(), + GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, i420ByteBuffer); + + GlUtil.checkNoGLES2Error("YuvConverter.convert"); + + // Restore normal framebuffer. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + + // Prepare Y, U, and V ByteBuffer slices. + final int yPos = 0; + final int uPos = yPos + stride * frameHeight; + // Rows of U and V alternate in the buffer, so V data starts after the first row of U. + final int vPos = uPos + stride / 2; + + i420ByteBuffer.position(yPos); + i420ByteBuffer.limit(yPos + stride * frameHeight); + final ByteBuffer dataY = i420ByteBuffer.slice(); + + i420ByteBuffer.position(uPos); + // The last row does not have padding. + final int uvSize = stride * (uvHeight - 1) + stride / 2; + i420ByteBuffer.limit(uPos + uvSize); + final ByteBuffer dataU = i420ByteBuffer.slice(); + + i420ByteBuffer.position(vPos); + i420ByteBuffer.limit(vPos + uvSize); + final ByteBuffer dataV = i420ByteBuffer.slice(); + + preparedBuffer.release(); + + return JavaI420Buffer.wrap(frameWidth, frameHeight, dataY, stride, dataU, stride, dataV, stride, + () -> { JniCommon.nativeFreeByteBuffer(i420ByteBuffer); }); + } + + public void release() { + threadChecker.checkIsOnValidThread(); + drawer.release(); + i420TextureFrameBuffer.release(); + videoFrameDrawer.release(); + // Allow this class to be reused. + threadChecker.detachThread(); + } +} |