From c669308ba80a03e97d6b276f6abc3d8c1b5d9718 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 17 Sep 2018 16:25:30 -0300 Subject: [PATCH] Fast snapshot with new GL preview. Take snapshots while taking video --- MIGRATION.md | 15 +- .../cameraview/EglBaseSurface.java | 41 ++- .../com/otaliastudios/cameraview/EglCore.java | 2 + .../otaliastudios/cameraview/EglViewport.java | 63 ++--- .../cameraview/EglWindowSurface.java | 8 + .../cameraview/VideoTextureEncoder.java | 2 +- .../com/otaliastudios/cameraview/Camera1.java | 7 +- .../cameraview/SnapshotPictureRecorder.java | 108 +++++++- .../cameraview/SnapshotVideoRecorder.java | 4 +- .../com/otaliastudios/cameraview/Preview.java | 6 +- .../cameraview/GLCameraPreview.java | 236 ++++++------------ 11 files changed, 284 insertions(+), 208 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 3c435233..1e399200 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -32,7 +32,7 @@ - VideoSizeSelector: added. It is needed to choose the capture size in VIDEO mode. Defaults to SizeSelectors.biggest(), but you can choose by aspect ratio or whatever. - isTakingPicture(): added on top of isTakingVideo(). -- takeVideoSnapshot(): new api. Requires the experimental flag, API 18 or it will throw. +- takeVideoSnapshot(): new api. API 18 and the Gl preview, or it will throw. Respects orientation, videocodec and max duration limit. Automatically rotates the data. Automatically crops the video. NO audio support. @@ -41,10 +41,11 @@ The default is GlSurfaceView and it is highly recommended that you do not change this. - New pictureRecorder interface for picture capturing. - Created FullPictureRecorder and SnapshotPictureRecorder for capturing HQ pictures and snapshots. +- When preview is GlSurface, the SnapshotPictureRecorder will use the gl texture and draw it into JPEG. + This is really fast and allows us to avoid RotationHelper, creating bitmap copies, OOMs, EXIF stuff. +- When preview is GlSurface, you can take snapshots while recording video (or video snapshots!). + TODO: document this +- TODO: cameraPreview documentation +- TODO: takeVideoSnapshot documentation -TODO: cameraPreview documentation -TODO: takeVideoSnapshot documentation - -TODO: add audio to the video snapshots -TODO: improve SnapshotPictureRecorder so that, if preview is GL, we catch the preview through GLES drawing - this would finally remove the RotationHelper and OOMs! \ No newline at end of file +TODO: add audio to the video snapshots \ No newline at end of file diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java index b4caa10b..f0c953a3 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java @@ -25,6 +25,7 @@ import android.support.annotation.RequiresApi; import android.util.Log; import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -157,7 +158,7 @@ class EglBaseSurface extends EglElement { *

* Expects that this object's EGL surface is current. */ - public void saveFrame(File file) throws IOException { + public void saveFrameToFile(File file) throws IOException { if (!mEglCore.isCurrent(mEGLSurface)) { throw new RuntimeException("Expected EGL context/surface is not current"); } @@ -196,4 +197,42 @@ class EglBaseSurface extends EglElement { if (bos != null) bos.close(); } } + + /** + * Saves the EGL surface to jpeg. + *

+ * Expects that this object's EGL surface is current. + */ + public byte[] saveFrameToJpeg() { + if (!mEglCore.isCurrent(mEGLSurface)) { + throw new RuntimeException("Expected EGL context/surface is not current"); + } + + // glReadPixels fills in a "direct" ByteBuffer with what is essentially big-endian RGBA + // data (i.e. a byte of red, followed by a byte of green...). While the Bitmap + // constructor that takes an int[] wants little-endian ARGB (blue/red swapped), the + // Bitmap "copy pixels" method wants the same format GL provides. + // + // Ideally we'd have some way to re-use the ByteBuffer, especially if we're calling + // here often. + // + // Making this even more interesting is the upside-down nature of GL, which means + // our output will look upside down relative to what appears on screen if the + // typical GL conventions are used. + + int width = getWidth(); + int height = getHeight(); + ByteBuffer buf = ByteBuffer.allocateDirect(width * height * 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf); + check("glReadPixels"); + buf.rewind(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(buf.array().length); + Bitmap bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + bmp.copyPixelsFromBuffer(buf); + bmp.compress(Bitmap.CompressFormat.JPEG, 90, bos); + bmp.recycle(); + return bos.toByteArray(); + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java index 2d1d527e..deb1cacf 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java @@ -28,6 +28,8 @@ import android.support.annotation.RequiresApi; import android.util.Log; import android.view.Surface; +import javax.microedition.khronos.egl.EGL10; + /** * -- from grafika -- * diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java index 7d1ffa4a..ca972e09 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java @@ -56,20 +56,21 @@ class EglViewport extends EglElement { }; // Stuff from Drawable2d.FULL_RECTANGLE - private FloatBuffer mVertexArray = floatBuffer(FULL_RECTANGLE_COORDS); - private FloatBuffer mTexCoordArray = floatBuffer(FULL_RECTANGLE_TEX_COORDS); + private FloatBuffer mVertexCoordinatesArray = floatBuffer(FULL_RECTANGLE_COORDS); + private FloatBuffer mTextureCoordinatesArray = floatBuffer(FULL_RECTANGLE_TEX_COORDS); private int mVertexCount = FULL_RECTANGLE_COORDS.length / 2; - private final int mCoordsPerVertex = 2; + private final int mCoordinatesPerVertex = 2; private final int mVertexStride = 8; - private final int mTexCoordStride = 8; + private final int mTextureStride = 8; // Stuff from Texture2dProgram private int mProgramHandle; private int mTextureTarget; - private int muMVPMatrixLoc; - private int muTexMatrixLoc; - private int maPositionLoc; - private int maTextureCoordLoc; + // Program attributes + private int muMVPMatrixLocation; + private int muTexMatrixLocation; + private int maPositionLocation; + private int maTextureCoordLocation; // private int muKernelLoc; // Used for filtering // private int muTexOffsetLoc; // Used for filtering @@ -78,14 +79,14 @@ class EglViewport extends EglElement { EglViewport() { mTextureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES; mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER); - maPositionLoc = GLES20.glGetAttribLocation(mProgramHandle, "aPosition"); - checkLocation(maPositionLoc, "aPosition"); - maTextureCoordLoc = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord"); - checkLocation(maTextureCoordLoc, "aTextureCoord"); - muMVPMatrixLoc = GLES20.glGetUniformLocation(mProgramHandle, "uMVPMatrix"); - checkLocation(muMVPMatrixLoc, "uMVPMatrix"); - muTexMatrixLoc = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix"); - checkLocation(muTexMatrixLoc, "uTexMatrix"); + maPositionLocation = GLES20.glGetAttribLocation(mProgramHandle, "aPosition"); + checkLocation(maPositionLocation, "aPosition"); + maTextureCoordLocation = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord"); + checkLocation(maTextureCoordLocation, "aTextureCoord"); + muMVPMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uMVPMatrix"); + checkLocation(muMVPMatrixLocation, "uMVPMatrix"); + muTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix"); + checkLocation(muTexMatrixLocation, "uTexMatrix"); // Stuff from Drawable2d.FULL_RECTANGLE @@ -122,15 +123,15 @@ class EglViewport extends EglElement { return texId; } - void drawFrame(int textureId, float[] texMatrix) { - drawFrame(textureId, texMatrix, - IDENTITY_MATRIX, mVertexArray, 0, - mVertexCount, mCoordsPerVertex, - mVertexStride, mTexCoordArray, - mTexCoordStride); + void drawFrame(int textureId, float[] textureMatrix) { + drawFrame(textureId, textureMatrix, + IDENTITY_MATRIX, mVertexCoordinatesArray, 0, + mVertexCount, mCoordinatesPerVertex, + mVertexStride, mTextureCoordinatesArray, + mTextureStride); } - private void drawFrame(int textureId, float[] texMatrix, + private void drawFrame(int textureId, float[] textureMatrix, float[] mvpMatrix, FloatBuffer vertexBuffer, int firstVertex, int vertexCount, int coordsPerVertex, int vertexStride, FloatBuffer texBuffer, int texStride) { @@ -145,28 +146,28 @@ class EglViewport extends EglElement { GLES20.glBindTexture(mTextureTarget, textureId); // Copy the model / view / projection matrix over. - GLES20.glUniformMatrix4fv(muMVPMatrixLoc, 1, false, mvpMatrix, 0); + GLES20.glUniformMatrix4fv(muMVPMatrixLocation, 1, false, mvpMatrix, 0); check("glUniformMatrix4fv"); // Copy the texture transformation matrix over. - GLES20.glUniformMatrix4fv(muTexMatrixLoc, 1, false, texMatrix, 0); + GLES20.glUniformMatrix4fv(muTexMatrixLocation, 1, false, textureMatrix, 0); check("glUniformMatrix4fv"); // Enable the "aPosition" vertex attribute. - GLES20.glEnableVertexAttribArray(maPositionLoc); + GLES20.glEnableVertexAttribArray(maPositionLocation); check("glEnableVertexAttribArray"); // Connect vertexBuffer to "aPosition". - GLES20.glVertexAttribPointer(maPositionLoc, coordsPerVertex, + GLES20.glVertexAttribPointer(maPositionLocation, coordsPerVertex, GLES20.GL_FLOAT, false, vertexStride, vertexBuffer); check("glVertexAttribPointer"); // Enable the "aTextureCoord" vertex attribute. - GLES20.glEnableVertexAttribArray(maTextureCoordLoc); + GLES20.glEnableVertexAttribArray(maTextureCoordLocation); check("glEnableVertexAttribArray"); // Connect texBuffer to "aTextureCoord". - GLES20.glVertexAttribPointer(maTextureCoordLoc, 2, + GLES20.glVertexAttribPointer(maTextureCoordLocation, 2, GLES20.GL_FLOAT, false, texStride, texBuffer); check("glVertexAttribPointer"); @@ -175,8 +176,8 @@ class EglViewport extends EglElement { check("glDrawArrays"); // Done -- disable vertex array, texture, and program. - GLES20.glDisableVertexAttribArray(maPositionLoc); - GLES20.glDisableVertexAttribArray(maTextureCoordLoc); + GLES20.glDisableVertexAttribArray(maPositionLocation); + GLES20.glDisableVertexAttribArray(maTextureCoordLocation); GLES20.glBindTexture(mTextureTarget, 0); GLES20.glUseProgram(0); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglWindowSurface.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglWindowSurface.java index 2a827f88..6385236c 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglWindowSurface.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglWindowSurface.java @@ -54,6 +54,14 @@ class EglWindowSurface extends EglBaseSurface { createWindowSurface(surfaceTexture); } + /** + * Associates an EGL surface with the Surface. + */ + public EglWindowSurface(EglCore eglCore, Surface surface) { + super(eglCore); + createWindowSurface(surface); + } + /** * Releases any resources associated with the EGL surface (and, if configured to do so, * with the Surface as well). diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoTextureEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoTextureEncoder.java index c68eb1f3..cdc0f197 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoTextureEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoTextureEncoder.java @@ -157,7 +157,7 @@ class VideoTextureEncoder implements Runnable { mInputWindowSurface = null; } if (mFullScreen != null) { - mFullScreen.release(false); + mFullScreen.release(true); mFullScreen = null; } if (mEglCore != null) { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java b/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java index 1b87cb07..fbf8d368 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java @@ -527,6 +527,8 @@ class Camera1 extends CameraController implements Camera.PreviewCallback, Camera @Override public void run() { if (mMode == Mode.VIDEO) { + // Could redirect to takePictureSnapshot, but it's better if people know + // what they are doing. throw new IllegalStateException("Can't take hq pictures while in VIDEO mode"); } @@ -550,11 +552,6 @@ class Camera1 extends CameraController implements Camera.PreviewCallback, Camera schedule(null, true, new Runnable() { @Override public void run() { - if (isTakingVideo()) { - // TODO v2: what to do here? - return; - } - LOG.v("takePictureSnapshot: performing.", isTakingPicture()); if (isTakingPicture()) return; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java index 46e26517..75bd8a74 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java @@ -1,12 +1,27 @@ package com.otaliastudios.cameraview; +import android.annotation.SuppressLint; +import android.graphics.Bitmap; import android.graphics.ImageFormat; import android.graphics.Rect; +import android.graphics.SurfaceTexture; import android.graphics.YuvImage; import android.hardware.Camera; +import android.opengl.EGL14; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.opengl.GLES11Ext; +import android.opengl.GLES20; import android.opengl.GLSurfaceView; +import android.opengl.Matrix; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; /** * A {@link PictureResult} that uses standard APIs. @@ -23,12 +38,12 @@ class SnapshotPictureRecorder extends PictureRecorder { private Size mSensorPreviewSize; private int mFormat; - SnapshotPictureRecorder(PictureResult stub, Camera1 controller, Camera camera, AspectRatio viewRatio) { + SnapshotPictureRecorder(PictureResult stub, Camera1 controller, Camera camera, AspectRatio outputRatio) { super(stub, controller); mController = controller; mPreview = controller.mPreview; mCamera = camera; - mOutputRatio = viewRatio; + mOutputRatio = outputRatio; mFormat = mController.mPreviewFormat; mSensorPreviewSize = mController.mPreviewSize; } @@ -42,11 +57,94 @@ class SnapshotPictureRecorder extends PictureRecorder { } } - private void takeGl(GLCameraPreview preview) { - // TODO implement. - takeLegacy(); + @SuppressLint("NewApi") + private void takeGl(final GLCameraPreview preview) { + preview.addRendererFrameCallback(new GLCameraPreview.RendererFrameCallback() { + + int mTextureId; + SurfaceTexture mSurfaceTexture; + float[] mTransform; + + public void onRendererTextureCreated(int textureId) { + mTextureId = textureId; + mSurfaceTexture = new SurfaceTexture(mTextureId, true); + // Need to crop the size. + Rect crop = CropHelper.computeCrop(mResult.size, mOutputRatio); + mResult.size = new Size(crop.width(), crop.height()); + mSurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); + mTransform = new float[16]; + } + + @Override + public void onRendererFrame(SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { + preview.removeRendererFrameCallback(this); + + // This kinda work but has drawbacks: + // - output is upside down due to coordinates in GL: need to flip the byte[] someway + // - output is not rotated as we would like to: need to create a bitmap copy... + // - works only in the renderer thread, where it allocates the buffer and reads pixels. Bad! + /* + ByteBuffer buffer = ByteBuffer.allocateDirect(width * height * 4); + buffer.order(ByteOrder.LITTLE_ENDIAN); + GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer); + buffer.rewind(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(buffer.array().length); + Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + bitmap.copyPixelsFromBuffer(buffer); + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, bos); + bitmap.recycle(); */ + + // For this reason it is better to create a new surface, + // and draw the last frame again there. + final EGLContext eglContext = EGL14.eglGetCurrentContext(); + final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE); + // final EGLSurface oldSurface = EGL14.eglGetCurrentSurface(EGL14.EGL_DRAW); + // final EGLDisplay oldDisplay = EGL14.eglGetCurrentDisplay(); + WorkerHandler.run(new Runnable() { + @Override + public void run() { + EglWindowSurface surface = new EglWindowSurface(core, mSurfaceTexture); + surface.makeCurrent(); + EglViewport viewport = new EglViewport(); + mSurfaceTexture.updateTexImage(); + mSurfaceTexture.getTransformMatrix(mTransform); + + // Apply scale and crop: + // NOTE: scaleX and scaleY are in REF_VIEW, while our input appears to be in REF_SENSOR. + boolean flip = mController.flip(CameraController.REF_VIEW, CameraController.REF_SENSOR); + float realScaleX = flip ? scaleY : scaleX; + float realScaleY = flip ? scaleX : scaleY; + float scaleTranslX = (1F - realScaleX) / 2F; + float scaleTranslY = (1F - realScaleY) / 2F; + Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0); + Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1); + + // Apply rotation: + // Not sure why we need the minus here... It makes no sense. + int rotation = -mResult.rotation; + mResult.rotation = 0; + Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mTransform, 0, rotation, 0, 0, 1); + Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); + + viewport.drawFrame(mTextureId, mTransform); + // don't - surface.swapBuffers(); + mResult.jpeg = surface.saveFrameToJpeg(); + mSurfaceTexture.releaseTexImage(); + + // EGL14.eglMakeCurrent(oldDisplay, oldSurface, oldSurface, eglContext); + surface.release(); + viewport.release(); + mSurfaceTexture.release(); + core.release(); + dispatchResult(); + } + }); + } + }); } + private void takeLegacy() { mCamera.setOneShotPreviewCallback(new Camera.PreviewCallback() { @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java index 8f25dda6..b56711fc 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java @@ -33,7 +33,7 @@ class SnapshotVideoRecorder extends VideoRecorder implements GLCameraPreview.Ren super(stub, listener); mEncoder = new VideoTextureEncoder(); mPreview = preview; - mPreview.setRendererFrameCallback(this); + mPreview.addRendererFrameCallback(this); } @Override @@ -109,7 +109,7 @@ class SnapshotVideoRecorder extends VideoRecorder implements GLCameraPreview.Ren mCurrentState = STATE_NOT_RECORDING; mEncoder = null; - mPreview.setRendererFrameCallback(null); + mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this); mPreview = null; } diff --git a/cameraview/src/main/options/com/otaliastudios/cameraview/Preview.java b/cameraview/src/main/options/com/otaliastudios/cameraview/Preview.java index b7c78d87..e111bbab 100644 --- a/cameraview/src/main/options/com/otaliastudios/cameraview/Preview.java +++ b/cameraview/src/main/options/com/otaliastudios/cameraview/Preview.java @@ -16,13 +16,15 @@ public enum Preview implements Control { /** * Preview engine based on {@link android.view.TextureView}. - * Stable, but does not support all features (like video snapshots). + * Stable, but does not support all features (like video snapshots, + * or picture snapshot while taking videos). */ TEXTURE(1), /** * Preview engine based on {@link android.opengl.GLSurfaceView}. - * This is the best engine available. + * This is the best engine available. Supports video snapshots, + * and picture snapshots while taking videos. */ GL_SURFACE(2); diff --git a/cameraview/src/main/views/com/otaliastudios/cameraview/GLCameraPreview.java b/cameraview/src/main/views/com/otaliastudios/cameraview/GLCameraPreview.java index 91bd3968..623bd182 100644 --- a/cameraview/src/main/views/com/otaliastudios/cameraview/GLCameraPreview.java +++ b/cameraview/src/main/views/com/otaliastudios/cameraview/GLCameraPreview.java @@ -11,7 +11,14 @@ import android.view.SurfaceHolder; import android.view.View; import android.view.ViewGroup; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import javax.microedition.khronos.egl.EGL10; import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLDisplay; +import javax.microedition.khronos.egl.EGLSurface; import javax.microedition.khronos.opengles.GL10; /** @@ -19,42 +26,43 @@ import javax.microedition.khronos.opengles.GL10; * * - in the SurfaceTexture constructor we pass the GL texture handle that we have created. * - * - The SurfaceTexture is linked to the Camera1 object. It will pass down buffers of data with - * a specified size (that is, the Camera1 preview size). - * - * - When SurfaceTexture.updateTexImage() is called, it will take the latest image from the camera stream - * and update it into the GL texture that was passed. + * - The SurfaceTexture is linked to the Camera1 object. The camera will pass down buffers of data with + * a specified size (that is, the Camera1 preview size). For this reason we don't have to specify + * surfaceTexture.setDefaultBufferSize() (like we do, for example, in SnapshotPictureRecorder). * - * - Now we have a GL texture referencing data. It must be drawn. - * [Note: it must be drawn using a transformation matrix taken from SurfaceTexture] + * - When SurfaceTexture.updateTexImage() is called, it will fetch the latest texture image from the + * camera stream and assign it to the GL texture that was passed. + * Now the GL texture must be drawn using draw* APIs. The SurfaceTexture will also give us + * the transformation matrix to be applied. * * - The easy way to render an OpenGL texture is using the {@link GLSurfaceView} class. - * It manages the gl context, hosts a surface and runs a separated rendering thread that will perform + * It manages the GL context, hosts a surface and runs a separated rendering thread that will perform * the rendering. * * - As per docs, we ask the GLSurfaceView to delegate rendering to us, using * {@link GLSurfaceView#setRenderer(GLSurfaceView.Renderer)}. We request a render on the SurfaceView * anytime the SurfaceTexture notifies that it has new data available (see OnFrameAvailableListener below). * - * - Everything is linked: + * - So in short: * - The SurfaceTexture has buffers of data of mInputStreamSize - * - The SurfaceView hosts a view (and surface) of size mOutputSurfaceSize + * - The SurfaceView hosts a view (and a surface) of size mOutputSurfaceSize. + * These are determined by the CameraView.onMeasure method. * - We have a GL rich texture to be drawn (in the given method & thread). * - * TODO - * CROPPING: Managed to do this using Matrix transformation. - * UPDATING: Managed to work using view.onPause and onResume. - * TAKING PICTURES: Sometime the snapshot takes ages... Can't reproduce anymore. Cool. - * TAKING VIDEOS: Still have not tried... + * This class will provide rendering callbacks to anyone who registers a {@link RendererFrameCallback}. + * Callbacks are guaranteed to be called on the renderer thread, which means that we can fetch + * the GL context that was created and is managed by the {@link GLSurfaceView}. */ class GLCameraPreview extends CameraPreview implements GLSurfaceView.Renderer { private boolean mDispatched; private final float[] mTransformMatrix = new float[16]; - private int mOutputTextureId = -1; + private int mOutputTextureId = 0; private SurfaceTexture mInputSurfaceTexture; private EglViewport mOutputViewport; - private RendererFrameCallback mRendererFrameCallback; + private Set mRendererFrameCallbacks = Collections.synchronizedSet(new HashSet()); + /* for tests */ float mScaleX = 1F; + /* for tests */ float mScaleY = 1F; GLCameraPreview(Context context, ViewGroup parent, SurfaceCallback callback) { super(context, parent, callback); @@ -97,10 +105,8 @@ class GLCameraPreview extends CameraPreview imple @Override void onDestroy() { super.onDestroy(); - releaseInputSurfaceTexture(); - } - - private void releaseInputSurfaceTexture() { + // View is gone, so EGL context is gone: callbacks make no sense anymore. + mRendererFrameCallbacks.clear(); if (mInputSurfaceTexture != null) { mInputSurfaceTexture.setOnFrameAvailableListener(null); mInputSurfaceTexture.release(); @@ -113,28 +119,31 @@ class GLCameraPreview extends CameraPreview imple } } - private void createInputSurfaceTexture() { + // Renderer thread + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { mOutputViewport = new EglViewport(); mOutputTextureId = mOutputViewport.createTexture(); mInputSurfaceTexture = new SurfaceTexture(mOutputTextureId); + getView().queueEvent(new Runnable() { + @Override + public void run() { + for (RendererFrameCallback callback : mRendererFrameCallbacks) { + callback.onRendererTextureCreated(mOutputTextureId); + } + } + }); // Since we are using GLSurfaceView.RENDERMODE_WHEN_DIRTY, we must notify the SurfaceView // of dirtyness, so that it draws again. This is how it's done. mInputSurfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() { @Override public void onFrameAvailable(SurfaceTexture surfaceTexture) { - // requestRender is thread-safe. - getView().requestRender(); + getView().requestRender(); // requestRender is thread-safe. } }); } - // Renderer thread - @Override - public void onSurfaceCreated(GL10 gl, EGLConfig config) { - createInputSurfaceTexture(); - } - // Renderer thread @SuppressWarnings("StatementWithEmptyBody") @Override @@ -143,27 +152,27 @@ class GLCameraPreview extends CameraPreview imple dispatchOnOutputSurfaceAvailable(width, height); mDispatched = true; } else if (mOutputSurfaceWidth == width && mOutputSurfaceHeight == height) { - // This change can be triggered by ourselves (see below). Ignore. + // I was experimenting and this was happening. + // Not sure if it is stil needed now. } else { // With other CameraPreview implementation we could just dispatch the 'size changed' event - // to the controller and everything would go straight. In case of GL, apparently we have to: - // - create a new texture (release the old) - // - unbind camera and surface - // - stop camera preview - // - recreate the GL context using view.onPause() and onResume() - // ... - onSizeChangeImplementation4(width, height); + // to the controller and everything would go straight. In case of GL, apparently we have to + // force recreate the EGLContext by calling onPause and onResume in the UI thread. + dispatchOnOutputSurfaceDestroyed(); + getView().post(new Runnable() { + @Override + public void run() { + getView().onPause(); + getView().onResume(); + dispatchOnOutputSurfaceAvailable(width, height); + } + }); } } - + // Renderer thread @Override public void onDrawFrame(GL10 gl) { - // This are only needed with some implementations, - // and implementation4 seems to work well without them. - // if (mInputSurfaceTexture == null) return; - // if (mOutputViewport == null) return; - // Latch the latest frame. If there isn't anything new, // we'll just re-use whatever was there before. mInputSurfaceTexture.updateTexImage(); @@ -172,24 +181,21 @@ class GLCameraPreview extends CameraPreview imple return; } - if (mRendererFrameCallback != null) { - mRendererFrameCallback.onRendererFrame(mInputSurfaceTexture, mScaleX, mScaleY); - } - - // Draw the video frame. mInputSurfaceTexture.getTransformMatrix(mTransformMatrix); if (isCropping()) { + // Scaling is easy. However: // If the view is 10x1000 (very tall), it will show only the left strip of the preview (not the center one). // If the view is 1000x10 (very large), it will show only the bottom strip of the preview (not the center one). - // We must use Matrix.translateM, and it must happen before the crop. + // So we must use Matrix.translateM, and it must happen before the crop. float translX = (1F - mScaleX) / 2F; float translY = (1F - mScaleY) / 2F; Matrix.translateM(mTransformMatrix, 0, translX, translY, 0); - - // Crop. Works, but without translation, it is not centered. Matrix.scaleM(mTransformMatrix, 0, mScaleX, mScaleY, 1); } mOutputViewport.drawFrame(mOutputTextureId, mTransformMatrix); + for (RendererFrameCallback callback : mRendererFrameCallbacks) { + callback.onRendererFrame(mInputSurfaceTexture, mScaleX, mScaleY); + } } @Override @@ -207,9 +213,6 @@ class GLCameraPreview extends CameraPreview imple return true; } - /* for tests */ float mScaleX = 1F; - /* for tests */ float mScaleY = 1F; - /** * To crop in GL, we could actually use view.setScaleX and setScaleY, but only from Android N onward. * See documentation: https://developer.android.com/reference/android/view/SurfaceView @@ -244,114 +247,39 @@ class GLCameraPreview extends CameraPreview imple mCropTask.end(null); } + interface RendererFrameCallback { - // This does work but looks like a lot of stuff. - private void onSizeChangeImplementation1(final int width, final int height) { - releaseInputSurfaceTexture(); - dispatchOnOutputSurfaceDestroyed(); - getView().post(new Runnable() { - @Override - public void run() { - getView().onPause(); - getView().onResume(); - getView().queueEvent(new Runnable() { - @Override - public void run() { - createInputSurfaceTexture(); - dispatchOnOutputSurfaceAvailable(width, height); - } - }); - } - }); - } - - // This does not work. We get: startPreview failed. - private void onSizeChangeImplementation2(final int width, final int height) { - releaseInputSurfaceTexture(); - getView().post(new Runnable() { - @Override - public void run() { - getView().onPause(); - getView().onResume(); - getView().queueEvent(new Runnable() { - @Override - public void run() { - createInputSurfaceTexture(); - dispatchOnOutputSurfaceSizeChanged(width, height); - } - }); - } - }); - } - - // Works! So we don't need to recreate the GL texture. - private void onSizeChangeImplementation3(final int width, final int height) { - dispatchOnOutputSurfaceDestroyed(); - getView().post(new Runnable() { - @Override - public void run() { - getView().onPause(); - getView().onResume(); - getView().queueEvent(new Runnable() { - @Override - public void run() { - dispatchOnOutputSurfaceAvailable(width, height); - } - }); - } - }); - } - - // Works! This is getting easy. - private void onSizeChangeImplementation4(final int width, final int height) { - dispatchOnOutputSurfaceDestroyed(); - getView().post(new Runnable() { - @Override - public void run() { - getView().onPause(); - getView().onResume(); - dispatchOnOutputSurfaceAvailable(width, height); - } - }); - } + /** + * Called on the renderer thread, hopefully only once, to notify that + * the texture was created (or to inform a new callback of the old texture). + * + * @param textureId the GL texture linked to the image stream + */ + void onRendererTextureCreated(int textureId); - // Does not work. onPause and onResume must be called on the UI thread. - // This make sense. - private void onSizeChangeImplementation5(final int width, final int height) { - dispatchOnOutputSurfaceDestroyed(); - getView().onPause(); - getView().onResume(); - dispatchOnOutputSurfaceAvailable(width, height); + /** + * Called on the renderer thread after each frame was drawn. + * You are not supposed to hold for too long onto this thread, because + * well, it is the rendering thread. + * + * @param surfaceTexture the texture to get transformation + * @param scaleX the scaleX (in REF_VIEW) value + * @param scaleY the scaleY (in REF_VIEW) value + */ + void onRendererFrame(SurfaceTexture surfaceTexture, float scaleX, float scaleY); } - // Does NOT work. The EGL context must be recreated - // for this to work out. - private void onSizeChangeImplementation6(final int width, final int height) { - dispatchOnOutputSurfaceDestroyed(); - getView().post(new Runnable() { + void addRendererFrameCallback(@NonNull final RendererFrameCallback callback) { + getView().queueEvent(new Runnable() { @Override public void run() { - getView().setPreserveEGLContextOnPause(true); - getView().onPause(); - getView().onResume(); - getView().setPreserveEGLContextOnPause(false); - dispatchOnOutputSurfaceAvailable(width, height); + mRendererFrameCallbacks.add(callback); + if (mOutputTextureId != 0) callback.onRendererTextureCreated(mOutputTextureId); } }); } - interface RendererFrameCallback { - // Renderer thread. - void onRendererTextureCreated(int textureId); - - // Renderer thread. - void onRendererFrame(SurfaceTexture surfaceTexture, float scaleX, float scaleY); - } - - void setRendererFrameCallback(@Nullable RendererFrameCallback callback) { - mRendererFrameCallback = callback; - if (mRendererFrameCallback != null && mOutputTextureId != 0) { - mRendererFrameCallback.onRendererTextureCreated(mOutputTextureId); - } + void removeRendererFrameCallback(@NonNull final RendererFrameCallback callback) { + mRendererFrameCallbacks.remove(callback); } }