From e1721bb77d247e519804ec58f12b0269debce659 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 16 Dec 2019 14:24:10 +0100 Subject: [PATCH] Frame Processing maxWidth, maxHeight and format (#704) * Create CameraEngine and CameraBaseEngine * Promote filters to stable - no experimental flag * Fix setSnapshotMaxWidth / Height bugs * Add setFrameProcessingMaxWidth and setFrameProcessingMaxHeight * Add setFrameProcessingMaxWidth and setFrameProcessingMaxHeight (docs) * Prepare Frame for Images, abstract FrameManager, create ByteBufferFrameManager * Fix tests * Fix unit tests * Send Images for Camera2 * Tests * Add CameraView.setFrameProcessingFormat(int), tests, docs * Add CameraOptions.getSupportedFrameProcessingFormats(), tests * Add CameraEngine support, integration tests * Fix demo app, add getFrameProcessingPoolSize * Fix tests * Fix tests --- .github/workflows/build.yml | 2 +- README.md | 3 + .../cameraview/CameraViewTest.java | 51 +- .../engine/Camera1IntegrationTest.java | 17 + .../engine/Camera2IntegrationTest.java | 7 + .../engine/CameraIntegrationTest.java | 53 +- .../cameraview/engine/MockCameraEngine.java | 28 +- .../engine/options/Camera1OptionsTest.java | 7 + .../frame/ByteBufferFrameManagerTest.java | 105 ++ .../cameraview/frame/FrameManagerTest.java | 98 +- .../internal/utils/ImageHelperTest.java | 100 -- .../cameraview/CameraOptions.java | 13 + .../otaliastudios/cameraview/CameraView.java | 191 +++- .../cameraview/engine/Camera1Engine.java | 82 +- .../cameraview/engine/Camera2Engine.java | 191 ++-- .../cameraview/engine/CameraBaseEngine.java | 927 ++++++++++++++++++ .../cameraview/engine/CameraEngine.java | 862 +++------------- .../engine/options/Camera1Options.java | 4 + .../engine/options/Camera2Options.java | 10 + .../frame/ByteBufferFrameManager.java | 162 +++ .../otaliastudios/cameraview/frame/Frame.java | 34 +- .../cameraview/frame/FrameManager.java | 191 ++-- .../cameraview/frame/ImageFrameManager.java | 30 + .../internal/utils/ImageHelper.java | 100 -- cameraview/src/main/res/values/attrs.xml | 4 + .../cameraview/frame/FrameTest.java | 40 +- .../cameraview/demo/CameraActivity.java | 36 +- .../otaliastudios/cameraview/demo/Option.java | 42 + docs/_posts/2018-12-20-frame-processing.md | 62 +- docs/_posts/2019-08-06-filters.md | 6 +- 30 files changed, 2054 insertions(+), 1404 deletions(-) create mode 100644 cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java delete mode 100644 cameraview/src/androidTest/java/com/otaliastudios/cameraview/internal/utils/ImageHelperTest.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/frame/ImageFrameManager.java delete mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e47af51..f4543c7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: with: java-version: 1.8 - name: Execute emulator tests - timeout-minutes: 20 + timeout-minutes: 30 uses: reactivecircus/android-emulator-runner@v2.2.0 with: api-level: ${{ matrix.EMULATOR_API }} diff --git a/README.md b/README.md index 3982db71..ec06f86b 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ Using CameraView is extremely simple: app:cameraVideoSizeAspectRatio="@string/video_ratio" app:cameraSnapshotMaxWidth="@integer/snapshot_max_width" app:cameraSnapshotMaxHeight="@integer/snapshot_max_height" + app:cameraFrameProcessingMaxWidth="@integer/processing_max_width" + app:cameraFrameProcessingMaxHeight="@integer/processing_max_height" + app:cameraFrameProcessingFormat="@integer/processing_format" app:cameraVideoBitRate="@integer/video_bit_rate" app:cameraAudioBitRate="@integer/audio_bit_rate" app:cameraGestureTap="none|autoFocus|takePicture" diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java index 40e55c99..1bc19bed 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java @@ -3,6 +3,7 @@ package com.otaliastudios.cameraview; import android.content.Context; import android.content.res.TypedArray; +import android.graphics.ImageFormat; import android.graphics.PointF; import android.location.Location; import androidx.annotation.NonNull; @@ -168,8 +169,13 @@ public class CameraViewTest extends BaseTest { assertEquals(cameraView.getLocation(), null); assertEquals(cameraView.getExposureCorrection(), 0f, 0f); assertEquals(cameraView.getZoom(), 0f, 0f); - assertEquals(cameraView.getVideoMaxDuration(), 0, 0); - assertEquals(cameraView.getVideoMaxSize(), 0, 0); + assertEquals(cameraView.getVideoMaxDuration(), 0); + assertEquals(cameraView.getVideoMaxSize(), 0); + assertEquals(cameraView.getSnapshotMaxWidth(), 0); + assertEquals(cameraView.getSnapshotMaxHeight(), 0); + assertEquals(cameraView.getFrameProcessingMaxWidth(), 0); + assertEquals(cameraView.getFrameProcessingMaxHeight(), 0); + assertEquals(cameraView.getFrameProcessingFormat(), 0); // Self managed GestureParser gestures = new GestureParser(empty); @@ -801,6 +807,30 @@ public class CameraViewTest extends BaseTest { assertEquals(cameraView.getPreviewFrameRate(), 60, 0); } + @Test + public void testSnapshotMaxSize() { + cameraView.setSnapshotMaxWidth(500); + assertEquals(500, cameraView.getSnapshotMaxWidth()); + cameraView.setSnapshotMaxHeight(700); + assertEquals(700, cameraView.getSnapshotMaxHeight()); + } + + @Test + public void testFrameProcessingMaxSize() { + cameraView.setFrameProcessingMaxWidth(500); + assertEquals(500, cameraView.getFrameProcessingMaxWidth()); + cameraView.setFrameProcessingMaxHeight(700); + assertEquals(700, cameraView.getFrameProcessingMaxHeight()); + } + + @Test + public void testFrameProcessingFormat() { + cameraView.setFrameProcessingFormat(ImageFormat.YUV_420_888); + assertEquals(ImageFormat.YUV_420_888, cameraView.getFrameProcessingFormat()); + cameraView.setFrameProcessingFormat(ImageFormat.YUV_422_888); + assertEquals(ImageFormat.YUV_422_888, cameraView.getFrameProcessingFormat()); + } + //endregion //region Lists of listeners and processors @@ -975,26 +1005,11 @@ public class CameraViewTest extends BaseTest { } //endregion - // TODO: test permissions //region Filter - @Test(expected = RuntimeException.class) - public void testSetFilter_notExperimental() { - cameraView.setExperimental(false); - cameraView.setFilter(Filters.AUTO_FIX.newInstance()); - } - - @Test - public void testSetFilter_notExperimental_noFilter() { - cameraView.setExperimental(false); - cameraView.setFilter(Filters.NONE.newInstance()); - // no exception thrown - } - @Test public void testSetFilter() { - cameraView.setExperimental(true); Filter filter = Filters.AUTO_FIX.newInstance(); cameraView.setFilter(filter); verify(mockPreview, times(1)).setFilter(filter); @@ -1002,4 +1017,6 @@ public class CameraViewTest extends BaseTest { //noinspection ResultOfMethodCallIgnored verify(mockPreview, times(1)).getCurrentFilter(); } + + //endregion } diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java index 41e2d3d7..2b03f522 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java @@ -1,6 +1,13 @@ package com.otaliastudios.cameraview.engine; +import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.CameraOptions; import com.otaliastudios.cameraview.controls.Engine; +import com.otaliastudios.cameraview.frame.Frame; +import com.otaliastudios.cameraview.frame.FrameProcessor; +import com.otaliastudios.cameraview.tools.Op; +import com.otaliastudios.cameraview.tools.Retry; +import com.otaliastudios.cameraview.tools.SdkExclude; import org.junit.Test; import org.junit.runner.RunWith; @@ -10,6 +17,10 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.LargeTest; import androidx.test.filters.RequiresDevice; +import java.util.Collection; + +import static org.junit.Assert.assertNotNull; + /** * These tests work great on real devices, and are the only way to test actual CameraEngine * implementation - we really need to open the camera device. @@ -31,4 +42,10 @@ public class Camera1IntegrationTest extends CameraIntegrationTest protected long getMeteringTimeoutMillis() { return Camera1Engine.AUTOFOCUS_END_DELAY_MILLIS; } + + @Override + public void testFrameProcessing_maxSize() { + // Camera1Engine does not support different sizes. + // super.testFrameProcessing_maxSize(); + } } diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java index a2218890..19d16c02 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java @@ -9,6 +9,7 @@ import com.otaliastudios.cameraview.controls.Engine; import com.otaliastudios.cameraview.engine.action.ActionHolder; import com.otaliastudios.cameraview.engine.action.BaseAction; +import org.junit.Test; import org.junit.runner.RunWith; import androidx.annotation.NonNull; @@ -76,4 +77,10 @@ public class Camera2IntegrationTest extends CameraIntegrationTest if (shouldOpen) closeSync(true); return result; } + + @Override + public void testFrameProcessing_freezeRelease() { + // Camera2 Frames are not freezable. + // super.testFrameProcessing_freezeRelease(); + } } diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java index 3248bd48..57500c1f 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java @@ -51,6 +51,7 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import java.io.File; +import java.util.Collection; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -70,7 +71,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public abstract class CameraIntegrationTest extends BaseTest { +public abstract class CameraIntegrationTest extends BaseTest { private final static CameraLogger LOG = CameraLogger.create(CameraIntegrationTest.class.getSimpleName()); private final static long DELAY = 8000; @@ -1043,6 +1044,27 @@ public abstract class CameraIntegrationTest extends Base assert15Frames(processor); } + @Test + @Retry(emulatorOnly = true) + @SdkExclude(maxSdkVersion = 22, emulatorOnly = true) + public void testFrameProcessing_maxSize() { + final int max = 600; + camera.setFrameProcessingMaxWidth(max); + camera.setFrameProcessingMaxHeight(max); + final Op sizeOp = new Op<>(); + camera.addFrameProcessor(new FrameProcessor() { + @Override + public void process(@NonNull Frame frame) { + sizeOp.controller().end(frame.getSize()); + } + }); + openSync(true); + Size size = sizeOp.await(2000); + assertNotNull(size); + assertTrue(size.getWidth() <= max); + assertTrue(size.getHeight() <= max); + } + @Test @Retry(emulatorOnly = true) @SdkExclude(maxSdkVersion = 22, emulatorOnly = true) @@ -1109,6 +1131,35 @@ public abstract class CameraIntegrationTest extends Base } } + @Test + @Retry(emulatorOnly = true) + @SdkExclude(maxSdkVersion = 22, emulatorOnly = true) + public void testFrameProcessing_format() { + CameraOptions o = openSync(true); + Collection formats = o.getSupportedFrameProcessingFormats(); + for (int format : formats) { + LOG.i("[TEST FRAME FORMAT]", "Testing", format, "..."); + Op op = testFrameProcessorFormat(format); + assertNotNull(op.await(DELAY)); + } + } + + @NonNull + private Op testFrameProcessorFormat(final int format) { + final Op op = new Op<>(); + camera.setFrameProcessingFormat(format); + camera.addFrameProcessor(new FrameProcessor() { + @Override + public void process(@NonNull Frame frame) { + if (frame.getFormat() == format) { + op.controller().start(); + op.controller().end(true); + } + } + }); + return op; + } + //endregion //region Overlays diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java index 2e6b3213..aa556810 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java @@ -13,6 +13,7 @@ import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.controls.Flash; import com.otaliastudios.cameraview.controls.PictureFormat; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.ByteBufferFrameManager; import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.controls.Hdr; @@ -27,7 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; -public class MockCameraEngine extends CameraEngine { +public class MockCameraEngine extends CameraBaseEngine { public boolean mPictureCaptured; public boolean mFocusStarted; @@ -83,7 +84,7 @@ public class MockCameraEngine extends CameraEngine { } public void setMockState(@NonNull CameraState state) { - Task change = mOrchestrator.scheduleStateChange(getState(), + Task change = getOrchestrator().scheduleStateChange(getState(), state, false, new Callable>() { @@ -109,7 +110,6 @@ public class MockCameraEngine extends CameraEngine { mExposureCorrectionChanged = true; } - @Override public void setFlash(@NonNull Flash flash) { mFlash = flash; @@ -135,6 +135,16 @@ public class MockCameraEngine extends CameraEngine { mPictureFormat = pictureFormat; } + @Override + public void setHasFrameProcessors(boolean hasFrameProcessors) { + mHasFrameProcessors = hasFrameProcessors; + } + + @Override + public void setFrameProcessingFormat(int format) { + mFrameProcessingFormat = format; + } + @Override public void takePicture(@NonNull PictureResult.Stub stub) { super.takePicture(stub); @@ -172,6 +182,12 @@ public class MockCameraEngine extends CameraEngine { return new ArrayList<>(); } + @NonNull + @Override + protected List getFrameProcessingAvailableSizes() { + return new ArrayList<>(); + } + @Override public void startAutoFocus(@Nullable Gesture gesture, @NonNull PointF point) { mFocusStarted = true; @@ -180,13 +196,11 @@ public class MockCameraEngine extends CameraEngine { @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, null); + return new ByteBufferFrameManager(2, null); } @Override - public void setPlaySounds(boolean playSounds) { - - } + public void setPlaySounds(boolean playSounds) { } @Override protected boolean collectCameraInfo(@NonNull Facing facing) { diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java index 8db1d61c..4dbbe0b4 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java @@ -1,6 +1,7 @@ package com.otaliastudios.cameraview.engine.options; +import android.graphics.ImageFormat; import android.hardware.Camera; import com.otaliastudios.cameraview.BaseTest; @@ -8,6 +9,7 @@ import com.otaliastudios.cameraview.CameraOptions; import com.otaliastudios.cameraview.controls.Audio; import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.controls.Flash; +import com.otaliastudios.cameraview.controls.PictureFormat; import com.otaliastudios.cameraview.engine.mappers.Camera1Mapper; import com.otaliastudios.cameraview.gesture.GestureAction; import com.otaliastudios.cameraview.controls.Grid; @@ -54,6 +56,11 @@ public class Camera1OptionsTest extends BaseTest { assertFalse(o.isZoomSupported()); assertEquals(o.getExposureCorrectionMaxValue(), 0f, 0); assertEquals(o.getExposureCorrectionMinValue(), 0f, 0); + // Static + assertEquals(1, o.getSupportedPictureFormats().size()); + assertTrue(o.getSupportedPictureFormats().contains(PictureFormat.JPEG)); + assertEquals(1, o.getSupportedFrameProcessingFormats().size()); + assertTrue(o.getSupportedFrameProcessingFormats().contains(ImageFormat.NV21)); } private Camera.Size mockCameraSize(int width, int height) { diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java new file mode 100644 index 00000000..d3f1242f --- /dev/null +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java @@ -0,0 +1,105 @@ +package com.otaliastudios.cameraview.frame; + + +import android.graphics.ImageFormat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.filters.SmallTest; + +import com.otaliastudios.cameraview.BaseTest; +import com.otaliastudios.cameraview.size.Size; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(AndroidJUnit4.class) +@SmallTest +public class ByteBufferFrameManagerTest extends BaseTest { + + private ByteBufferFrameManager.BufferCallback callback; + + @Before + public void setUp() { + callback = mock(ByteBufferFrameManager.BufferCallback.class); + } + + @After + public void tearDown() { + callback = null; + } + + @Test + public void testAllocate() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + verify(callback, times(1)).onBufferAvailable(any(byte[].class)); + reset(callback); + + manager = new ByteBufferFrameManager(5, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + verify(callback, times(5)).onBufferAvailable(any(byte[].class)); + } + + @Test + public void testOnFrameReleased_alreadyFull() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + Frame frame1 = manager.getFrame(new byte[length], 0, 0); + // Since frame1 is already taken and poolSize = 1, a new Frame is created. + Frame frame2 = manager.getFrame(new byte[length], 0, 0); + // Release the first frame so it goes back into the pool. + manager.onFrameReleased(frame1, (byte[]) frame1.getData()); + reset(callback); + // Release the second. The pool is already full, so onBufferAvailable should not be called + // since this Frame instance will NOT be reused. + manager.onFrameReleased(frame2, (byte[]) frame2.getData()); + verify(callback, never()).onBufferAvailable((byte[]) frame2.getData()); + } + + @Test + public void testOnFrameReleased_sameLength() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + // A camera preview frame comes. Request a frame. + byte[] picture = new byte[length]; + Frame frame = manager.getFrame(picture, 0, 0); + + // Release the frame and ensure that onBufferAvailable is called. + reset(callback); + manager.onFrameReleased(frame, (byte[]) frame.getData()); + verify(callback, times(1)).onBufferAvailable(picture); + } + + @Test + public void testOnFrameReleased_differentLength() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + // A camera preview frame comes. Request a frame. + byte[] picture = new byte[length]; + Frame frame = manager.getFrame(picture, 0, 0); + + // Don't release the frame. Change the allocation size. + manager.setUp(ImageFormat.NV16, new Size(15, 15)); + + // Now release the old frame and ensure that onBufferAvailable is NOT called, + // because the released data has wrong length. + manager.onFrameReleased(frame, (byte[]) frame.getData()); + reset(callback); + verify(callback, never()).onBufferAvailable(picture); + } +} diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java index 77375047..b553e145 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java @@ -3,116 +3,44 @@ package com.otaliastudios.cameraview.frame; import android.graphics.ImageFormat; +import androidx.annotation.NonNull; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; import com.otaliastudios.cameraview.BaseTest; import com.otaliastudios.cameraview.size.Size; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @RunWith(AndroidJUnit4.class) @SmallTest public class FrameManagerTest extends BaseTest { - private FrameManager.BufferCallback callback; - - @Before - public void setUp() { - callback = mock(FrameManager.BufferCallback.class); - } - - @After - public void tearDown() { - callback = null; - } - - @Test - public void testAllocate() { - FrameManager manager = new FrameManager(1, callback); - manager.setUp(ImageFormat.NV21, new Size(50, 50)); - verify(callback, times(1)).onBufferAvailable(any(byte[].class)); - reset(callback); - - manager = new FrameManager(5, callback); - manager.setUp(ImageFormat.NV21, new Size(50, 50)); - verify(callback, times(5)).onBufferAvailable(any(byte[].class)); - } - @Test public void testFrameRecycling() { // A 1-pool manager will always recycle the same frame. - FrameManager manager = new FrameManager(1, callback); + FrameManager manager = new FrameManager(1, String.class) { + @Override + protected void onFrameDataReleased(@NonNull String data, boolean recycled) { } + + @NonNull + @Override + protected String onCloneFrameData(@NonNull String data) { + return data; + } + }; manager.setUp(ImageFormat.NV21, new Size(50, 50)); - Frame first = manager.getFrame(null, 0, 0); + Frame first = manager.getFrame("foo", 0, 0); first.release(); - - Frame second = manager.getFrame(null, 0, 0); + Frame second = manager.getFrame("bar", 0, 0); second.release(); - assertEquals(first, second); } - - @Test - public void testOnFrameReleased_alreadyFull() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - Frame frame1 = manager.getFrame(new byte[length], 0, 0); - // Since frame1 is already taken and poolSize = 1, a new Frame is created. - Frame frame2 = manager.getFrame(new byte[length], 0, 0); - // Release the first frame so it goes back into the pool. - manager.onFrameReleased(frame1, frame1.getData()); - reset(callback); - // Release the second. The pool is already full, so onBufferAvailable should not be called - // since this Frame instance will NOT be reused. - manager.onFrameReleased(frame2, frame2.getData()); - verify(callback, never()).onBufferAvailable(frame2.getData()); - } - - @Test - public void testOnFrameReleased_sameLength() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - // A camera preview frame comes. Request a frame. - byte[] picture = new byte[length]; - Frame frame = manager.getFrame(picture, 0, 0); - - // Release the frame and ensure that onBufferAvailable is called. - reset(callback); - manager.onFrameReleased(frame, frame.getData()); - verify(callback, times(1)).onBufferAvailable(picture); - } - - @Test - public void testOnFrameReleased_differentLength() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - // A camera preview frame comes. Request a frame. - byte[] picture = new byte[length]; - Frame frame = manager.getFrame(picture, 0, 0); - - // Don't release the frame. Change the allocation size. - manager.setUp(ImageFormat.NV16, new Size(15, 15)); - - // Now release the old frame and ensure that onBufferAvailable is NOT called, - // because the released data has wrong length. - manager.onFrameReleased(frame, frame.getData()); - reset(callback); - verify(callback, never()).onBufferAvailable(picture); - } } diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/internal/utils/ImageHelperTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/internal/utils/ImageHelperTest.java deleted file mode 100644 index a75f8438..00000000 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/internal/utils/ImageHelperTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.otaliastudios.cameraview.internal.utils; - - -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.graphics.Canvas; -import android.graphics.Color; -import android.graphics.ImageFormat; -import android.graphics.Paint; -import android.graphics.PorterDuff; -import android.graphics.Rect; -import android.graphics.YuvImage; -import android.media.Image; -import android.media.ImageReader; -import android.os.Handler; -import android.os.Looper; -import android.view.Surface; - -import androidx.annotation.NonNull; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import androidx.test.filters.SmallTest; - -import com.otaliastudios.cameraview.BaseTest; -import com.otaliastudios.cameraview.tools.Op; -import com.otaliastudios.cameraview.tools.SdkExclude; - -import org.junit.Test; -import org.junit.runner.RunWith; - -import java.io.ByteArrayOutputStream; - -import static org.junit.Assert.assertNotNull; - -/** - * Starting from API 29, surface.lockCanvas() sets the surface format to RGBA_8888: - * https://github.com/aosp-mirror/platform_frameworks_base/blob/android10-release/core/jni/android_view_Surface.cpp#L215-L217 . - * For this reason, acquireLatestImage crashes because we requested a different format. - */ -@SdkExclude(minSdkVersion = 29) -@RunWith(AndroidJUnit4.class) -@SmallTest -public class ImageHelperTest extends BaseTest { - - @NonNull - private Image getImage() { - ImageReader reader = ImageReader.newInstance(100, 100, ImageFormat.YUV_420_888, 1); - Surface readerSurface = reader.getSurface(); - final Op imageOp = new Op<>(); - reader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() { - @Override - public void onImageAvailable(ImageReader reader) { - Image image = reader.acquireLatestImage(); - if (image != null) imageOp.controller().end(image); - } - }, new Handler(Looper.getMainLooper())); - - // Write on reader surface. - Canvas readerCanvas = readerSurface.lockCanvas(null); - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); - paint.setColor(Color.RED); - readerCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.MULTIPLY); - readerCanvas.drawCircle(50, 50, 50, paint); - readerSurface.unlockCanvasAndPost(readerCanvas); - - // Wait - Image image = imageOp.await(5000); - assertNotNull(image); - return image; - } - - @Test - public void testImage() { - Image image = getImage(); - int width = image.getWidth(); - int height = image.getHeight(); - int bitsPerPixel = ImageFormat.getBitsPerPixel(ImageFormat.NV21); - int sizeBits = width * height * bitsPerPixel; - int sizeBytes = (int) Math.ceil(sizeBits / 8.0d); - byte[] bytes = new byte[sizeBytes]; - ImageHelper.convertToNV21(image, bytes); - image.close(); - - // Read the image - YuvImage yuvImage = new YuvImage(bytes, ImageFormat.NV21, width, height, null); - ByteArrayOutputStream jpegStream = new ByteArrayOutputStream(); - yuvImage.compressToJpeg(new Rect(0, 0, width, height), 100, jpegStream); - byte[] jpegByteArray = jpegStream.toByteArray(); - Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.length); - assertNotNull(bitmap); - - // Wanted to do assertions on the color here but it doesn't work. There must be an issue - // with how we are drawing the image in this test, since in real camera, the algorithm works well. - // So for now let's just test that nothing crashes during this process. - // int color = bitmap.getPixel(bitmap.getWidth() - 1, bitmap.getHeight() - 1); - // assertEquals(Color.red(color), 255, 5); - // assertEquals(Color.green(color), 0, 5); - // assertEquals(Color.blue(color), 0, 5); - // assertEquals(Color.alpha(color), 0, 5); - } -} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraOptions.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraOptions.java index 9095412d..c6b9f15a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraOptions.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraOptions.java @@ -56,6 +56,7 @@ public abstract class CameraOptions { protected Set supportedPictureAspectRatio = new HashSet<>(4); protected Set supportedVideoAspectRatio = new HashSet<>(3); protected Set supportedPictureFormats = new HashSet<>(2); + protected Set supportedFrameProcessingFormats = new HashSet<>(2); protected boolean zoomSupported; protected boolean exposureCorrectionSupported; @@ -237,6 +238,18 @@ public abstract class CameraOptions { return Collections.unmodifiableSet(supportedPictureFormats); } + /** + * Set of supported formats for frame processing, + * as {@link ImageFormat} constants. + * + * @see CameraView#setFrameProcessingFormat(int) + * @return a collection of supported values. + */ + @NonNull + public final Collection getSupportedFrameProcessingFormats() { + return Collections.unmodifiableSet(supportedFrameProcessingFormats); + } + /** * Whether zoom is supported. If this is false, pinch-to-zoom * will not work and {@link CameraView#setZoom(float)} will have no effect. diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java index f7147f5a..636bc98a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java @@ -201,6 +201,11 @@ public class CameraView extends FrameLayout implements LifecycleObserver { boolean pictureSnapshotMetering = a.getBoolean( R.styleable.CameraView_cameraPictureSnapshotMetering, DEFAULT_PICTURE_SNAPSHOT_METERING); + int snapshotMaxWidth = a.getInteger(R.styleable.CameraView_cameraSnapshotMaxWidth, 0); + int snapshotMaxHeight = a.getInteger(R.styleable.CameraView_cameraSnapshotMaxHeight, 0); + int frameMaxWidth = a.getInteger(R.styleable.CameraView_cameraFrameProcessingMaxWidth, 0); + int frameMaxHeight = a.getInteger(R.styleable.CameraView_cameraFrameProcessingMaxHeight, 0); + int frameFormat = a.getInteger(R.styleable.CameraView_cameraFrameProcessingFormat, 0); // Size selectors and gestures SizeSelectorParser sizeSelectors = new SizeSelectorParser(a); @@ -257,6 +262,12 @@ public class CameraView extends FrameLayout implements LifecycleObserver { setVideoBitRate(videoBitRate); setAutoFocusResetDelay(autoFocusResetDelay); setPreviewFrameRate(videoFrameRate); + setSnapshotMaxWidth(snapshotMaxWidth); + setSnapshotMaxHeight(snapshotMaxHeight); + setFrameProcessingMaxWidth(frameMaxWidth); + setFrameProcessingMaxHeight(frameMaxHeight); + setFrameProcessingFormat(frameFormat); + mCameraEngine.setHasFrameProcessors(!mFrameProcessors.isEmpty()); // Apply gestures mapGesture(Gesture.TAP, gestures.getTapAction()); @@ -667,7 +678,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver { break; case FILTER_CONTROL_1: - if (!mExperimental) break; if (getFilter() instanceof OneParameterFilter) { OneParameterFilter filter = (OneParameterFilter) getFilter(); oldValue = filter.getParameter1(); @@ -679,7 +689,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver { break; case FILTER_CONTROL_2: - if (!mExperimental) break; if (getFilter() instanceof TwoParameterFilter) { TwoParameterFilter filter = (TwoParameterFilter) getFilter(); oldValue = filter.getParameter2(); @@ -968,6 +977,11 @@ public class CameraView extends FrameLayout implements LifecycleObserver { setVideoBitRate(oldEngine.getVideoBitRate()); setAutoFocusResetDelay(oldEngine.getAutoFocusResetDelay()); setPreviewFrameRate(oldEngine.getPreviewFrameRate()); + setSnapshotMaxWidth(oldEngine.getSnapshotMaxWidth()); + setSnapshotMaxHeight(oldEngine.getSnapshotMaxHeight()); + setFrameProcessingMaxWidth(oldEngine.getFrameProcessingMaxWidth()); + setFrameProcessingMaxHeight(oldEngine.getFrameProcessingMaxHeight()); + setFrameProcessingFormat(0 /* this is very engine specific, so do not pass */); } /** @@ -1555,47 +1569,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver { mListeners.clear(); } - /** - * Adds a {@link FrameProcessor} instance to be notified of - * new frames in the preview stream. - * - * @param processor a frame processor. - */ - public void addFrameProcessor(@Nullable FrameProcessor processor) { - if (processor != null) { - mFrameProcessors.add(processor); - if (mFrameProcessors.size() == 1) { - mCameraEngine.setHasFrameProcessors(true); - } - } - } - - /** - * Remove a {@link FrameProcessor} that was previously registered. - * - * @param processor a frame processor - */ - public void removeFrameProcessor(@Nullable FrameProcessor processor) { - if (processor != null) { - mFrameProcessors.remove(processor); - if (mFrameProcessors.size() == 0) { - mCameraEngine.setHasFrameProcessors(false); - } - } - } - - /** - * Clears the list of {@link FrameProcessor} that have been registered - * to preview frames. - */ - public void clearFrameProcessors() { - boolean had = mFrameProcessors.size() > 0; - mFrameProcessors.clear(); - if (had) { - mCameraEngine.setHasFrameProcessors(false); - } - } - /** * Asks the camera to capture an image of the current scene. * This will trigger {@link CameraListener#onPictureTaken(PictureResult)} if a listener @@ -1767,6 +1740,24 @@ public class CameraView extends FrameLayout implements LifecycleObserver { mCameraEngine.setSnapshotMaxHeight(maxHeight); } + /** + * The max width for snapshots. + * @see #setSnapshotMaxWidth(int) + * @return max width + */ + public int getSnapshotMaxWidth() { + return mCameraEngine.getSnapshotMaxWidth(); + } + + /** + * The max height for snapshots. + * @see #setSnapshotMaxHeight(int) + * @return max height + */ + public int getSnapshotMaxHeight() { + return mCameraEngine.getSnapshotMaxHeight(); + } + /** * Returns the size used for snapshots, or null if it hasn't been computed * (for example if the surface is not ready). This is the preview size, rotated to match @@ -2277,6 +2268,110 @@ public class CameraView extends FrameLayout implements LifecycleObserver { //endregion + //region Frame Processing + + /** + * Adds a {@link FrameProcessor} instance to be notified of + * new frames in the preview stream. + * + * @param processor a frame processor. + */ + public void addFrameProcessor(@Nullable FrameProcessor processor) { + if (processor != null) { + mFrameProcessors.add(processor); + if (mFrameProcessors.size() == 1) { + mCameraEngine.setHasFrameProcessors(true); + } + } + } + + /** + * Remove a {@link FrameProcessor} that was previously registered. + * + * @param processor a frame processor + */ + public void removeFrameProcessor(@Nullable FrameProcessor processor) { + if (processor != null) { + mFrameProcessors.remove(processor); + if (mFrameProcessors.size() == 0) { + mCameraEngine.setHasFrameProcessors(false); + } + } + } + + /** + * Clears the list of {@link FrameProcessor} that have been registered + * to preview frames. + */ + public void clearFrameProcessors() { + boolean had = mFrameProcessors.size() > 0; + mFrameProcessors.clear(); + if (had) { + mCameraEngine.setHasFrameProcessors(false); + } + } + + /** + * Sets the max width for frame processing {@link Frame}s. + * This option is only supported by {@link Engine#CAMERA2} and will have no effect + * on other engines. + * + * @param maxWidth max width for frames + */ + public void setFrameProcessingMaxWidth(int maxWidth) { + mCameraEngine.setFrameProcessingMaxWidth(maxWidth); + } + + /** + * Sets the max height for frame processing {@link Frame}s. + * This option is only supported by {@link Engine#CAMERA2} and will have no effect + * on other engines. + * + * @param maxHeight max height for frames + */ + public void setFrameProcessingMaxHeight(int maxHeight) { + mCameraEngine.setFrameProcessingMaxHeight(maxHeight); + } + + /** + * The max width for frame processing frames. + * @see #setFrameProcessingMaxWidth(int) + * @return max width + */ + public int getFrameProcessingMaxWidth() { + return mCameraEngine.getFrameProcessingMaxWidth(); + } + + /** + * The max height for frame processing frames. + * @see #setFrameProcessingMaxHeight(int) + * @return max height + */ + public int getFrameProcessingMaxHeight() { + return mCameraEngine.getFrameProcessingMaxHeight(); + } + + /** + * Sets the {@link android.graphics.ImageFormat} for frame processing. + * Before applying you should check {@link CameraOptions#getSupportedFrameProcessingFormats()}. + * + * @param format image format + */ + public void setFrameProcessingFormat(int format) { + mCameraEngine.setFrameProcessingFormat(format); + } + + /** + * Returns the current frame processing format. + * @see #setFrameProcessingFormat(int) + * @return image format + */ + public int getFrameProcessingFormat() { + return mCameraEngine.getFrameProcessingFormat(); + } + + //endregion + //region Overlays @Override @@ -2333,11 +2428,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver { } else { boolean isNoFilter = filter instanceof NoFilter; boolean isFilterPreview = mCameraPreview instanceof FilterCameraPreview; - // If not experimental, we only allow NoFilter (called on creation). - if (!isNoFilter && !mExperimental) { - throw new RuntimeException("Filters are an experimental features and" + - " need the experimental flag set."); - } // If not a filter preview, we only allow NoFilter (called on creation). if (!isNoFilter && !isFilterPreview) { throw new RuntimeException("Filters are only supported by the GL_SURFACE preview." + @@ -2362,10 +2452,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver { */ @NonNull public Filter getFilter() { - if (!mExperimental) { - throw new RuntimeException("Filters are an experimental features and need " + - "the experimental flag set."); - } else if (mCameraPreview == null) { + if (mCameraPreview == null) { return mPendingFilter; } else if (mCameraPreview instanceof FilterCameraPreview) { return ((FilterCameraPreview) mCameraPreview).getCurrentFilter(); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java index 5adf6e2b..cc668bce 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java @@ -18,7 +18,6 @@ import android.view.SurfaceHolder; import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.Tasks; import com.otaliastudios.cameraview.CameraException; -import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.CameraOptions; import com.otaliastudios.cameraview.controls.PictureFormat; import com.otaliastudios.cameraview.engine.mappers.Camera1Mapper; @@ -26,6 +25,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis; import com.otaliastudios.cameraview.engine.offset.Reference; import com.otaliastudios.cameraview.engine.options.Camera1Options; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.ByteBufferFrameManager; import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.VideoResult; @@ -48,13 +48,14 @@ import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -public class Camera1Engine extends CameraEngine implements +public class Camera1Engine extends CameraBaseEngine implements Camera.PreviewCallback, Camera.ErrorCallback, - FrameManager.BufferCallback { + ByteBufferFrameManager.BufferCallback { private static final String JOB_FOCUS_RESET = "focus reset"; private static final String JOB_FOCUS_END = "focus end"; @@ -105,6 +106,15 @@ public class Camera1Engine extends CameraEngine implements return result; } + @EngineThread + @NonNull + @Override + protected List getFrameProcessingAvailableSizes() { + // We don't choose the frame processing size. + // It comes from the preview stream. + return Collections.singletonList(mPreviewStreamSize); + } + @EngineThread @Override protected void onPreviewStreamSizeChanged() { @@ -189,7 +199,7 @@ public class Camera1Engine extends CameraEngine implements @Override protected Task onStartPreview() { LOG.i("onStartPreview", "Dispatching onCameraPreviewStreamSizeChanged."); - mCallback.onCameraPreviewStreamSizeChanged(); + getCallback().onCameraPreviewStreamSizeChanged(); Size previewSize = getPreviewStreamSize(Reference.VIEW); if (previewSize == null) { @@ -286,8 +296,8 @@ public class Camera1Engine extends CameraEngine implements @Override protected Task onStopEngine() { LOG.i("onStopEngine:", "About to clean up."); - mOrchestrator.remove(JOB_FOCUS_RESET); - mOrchestrator.remove(JOB_FOCUS_END); + getOrchestrator().remove(JOB_FOCUS_RESET); + getOrchestrator().remove(JOB_FOCUS_END); if (mCamera != null) { try { LOG.i("onStopEngine:", "Clean up.", "Releasing camera."); @@ -473,7 +483,7 @@ public class Camera1Engine extends CameraEngine implements public void setFlash(@NonNull Flash flash) { final Flash old = mFlash; mFlash = flash; - mFlashTask = mOrchestrator.scheduleStateful("flash (" + flash + ")", + mFlashTask = getOrchestrator().scheduleStateful("flash (" + flash + ")", CameraState.ENGINE, new Runnable() { @Override @@ -497,7 +507,7 @@ public class Camera1Engine extends CameraEngine implements public void setLocation(@Nullable Location location) { final Location oldLocation = mLocation; mLocation = location; - mLocationTask = mOrchestrator.scheduleStateful("location", + mLocationTask = getOrchestrator().scheduleStateful("location", CameraState.ENGINE, new Runnable() { @Override @@ -524,7 +534,7 @@ public class Camera1Engine extends CameraEngine implements public void setWhiteBalance(@NonNull WhiteBalance whiteBalance) { final WhiteBalance old = mWhiteBalance; mWhiteBalance = whiteBalance; - mWhiteBalanceTask = mOrchestrator.scheduleStateful( + mWhiteBalanceTask = getOrchestrator().scheduleStateful( "white balance (" + whiteBalance + ")", CameraState.ENGINE, new Runnable() { @@ -554,7 +564,7 @@ public class Camera1Engine extends CameraEngine implements public void setHdr(@NonNull Hdr hdr) { final Hdr old = mHdr; mHdr = hdr; - mHdrTask = mOrchestrator.scheduleStateful("hdr (" + hdr + ")", + mHdrTask = getOrchestrator().scheduleStateful("hdr (" + hdr + ")", CameraState.ENGINE, new Runnable() { @Override @@ -578,7 +588,7 @@ public class Camera1Engine extends CameraEngine implements public void setZoom(final float zoom, @Nullable final PointF[] points, final boolean notify) { final float old = mZoomValue; mZoomValue = zoom; - mZoomTask = mOrchestrator.scheduleStateful("zoom (" + zoom + ")", + mZoomTask = getOrchestrator().scheduleStateful("zoom (" + zoom + ")", CameraState.ENGINE, new Runnable() { @Override @@ -587,7 +597,7 @@ public class Camera1Engine extends CameraEngine implements if (applyZoom(params, old)) { mCamera.setParameters(params); if (notify) { - mCallback.dispatchOnZoomChanged(mZoomValue, points); + getCallback().dispatchOnZoomChanged(mZoomValue, points); } } } @@ -610,7 +620,7 @@ public class Camera1Engine extends CameraEngine implements @Nullable final PointF[] points, final boolean notify) { final float old = mExposureCorrectionValue; mExposureCorrectionValue = EVvalue; - mExposureCorrectionTask = mOrchestrator.scheduleStateful( + mExposureCorrectionTask = getOrchestrator().scheduleStateful( "exposure correction (" + EVvalue + ")", CameraState.ENGINE, new Runnable() { @@ -620,7 +630,7 @@ public class Camera1Engine extends CameraEngine implements if (applyExposureCorrection(params, old)) { mCamera.setParameters(params); if (notify) { - mCallback.dispatchOnExposureCorrectionChanged(mExposureCorrectionValue, + getCallback().dispatchOnExposureCorrectionChanged(mExposureCorrectionValue, bounds, points); } } @@ -651,7 +661,7 @@ public class Camera1Engine extends CameraEngine implements public void setPlaySounds(boolean playSounds) { final boolean old = mPlaySounds; mPlaySounds = playSounds; - mPlaySoundsTask = mOrchestrator.scheduleStateful( + mPlaySoundsTask = getOrchestrator().scheduleStateful( "play sounds (" + playSounds + ")", CameraState.ENGINE, new Runnable() { @@ -688,7 +698,7 @@ public class Camera1Engine extends CameraEngine implements public void setPreviewFrameRate(float previewFrameRate) { final float old = previewFrameRate; mPreviewFrameRate = previewFrameRate; - mPreviewFrameRateTask = mOrchestrator.scheduleStateful( + mPreviewFrameRateTask = getOrchestrator().scheduleStateful( "preview fps (" + previewFrameRate + ")", CameraState.ENGINE, new Runnable() { @@ -748,7 +758,25 @@ public class Camera1Engine extends CameraEngine implements @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, this); + return new ByteBufferFrameManager(2, this); + } + + @NonNull + @Override + public ByteBufferFrameManager getFrameManager() { + return (ByteBufferFrameManager) super.getFrameManager(); + } + + @Override + public void setHasFrameProcessors(boolean hasFrameProcessors) { + // we don't care, FP is always on + mHasFrameProcessors = hasFrameProcessors; + } + + @Override + public void setFrameProcessingFormat(int format) { + // Ignore input: we only support NV21. + mFrameProcessingFormat = ImageFormat.NV21; } @Override @@ -768,7 +796,7 @@ public class Camera1Engine extends CameraEngine implements Frame frame = getFrameManager().getFrame(data, System.currentTimeMillis(), getAngles().offset(Reference.SENSOR, Reference.OUTPUT, Axis.RELATIVE_TO_SENSOR)); - mCallback.dispatchFrame(frame); + getCallback().dispatchFrame(frame); } //endregion @@ -786,7 +814,7 @@ public class Camera1Engine extends CameraEngine implements } final int viewWidthF = viewWidth; final int viewHeightF = viewHeight; - mOrchestrator.scheduleStateful("auto focus", CameraState.ENGINE, new Runnable() { + getOrchestrator().scheduleStateful("auto focus", CameraState.ENGINE, new Runnable() { @Override public void run() { if (!mCameraOptions.isAutoFocusSupported()) return; @@ -805,16 +833,16 @@ public class Camera1Engine extends CameraEngine implements if (maxAE > 0) params.setMeteringAreas(maxAE > 1 ? meteringAreas2 : meteringAreas1); params.setFocusMode(Camera.Parameters.FOCUS_MODE_AUTO); mCamera.setParameters(params); - mCallback.dispatchOnFocusStart(gesture, p); + getCallback().dispatchOnFocusStart(gesture, p); // The auto focus callback is not guaranteed to be called, but we really want it // to be. So we remove the old runnable if still present and post a new one. - mOrchestrator.remove(JOB_FOCUS_END); - mOrchestrator.scheduleDelayed(JOB_FOCUS_END, AUTOFOCUS_END_DELAY_MILLIS, + getOrchestrator().remove(JOB_FOCUS_END); + getOrchestrator().scheduleDelayed(JOB_FOCUS_END, AUTOFOCUS_END_DELAY_MILLIS, new Runnable() { @Override public void run() { - mCallback.dispatchOnFocusEnd(gesture, false, p); + getCallback().dispatchOnFocusEnd(gesture, false, p); } }); @@ -824,11 +852,11 @@ public class Camera1Engine extends CameraEngine implements mCamera.autoFocus(new Camera.AutoFocusCallback() { @Override public void onAutoFocus(boolean success, Camera camera) { - mOrchestrator.remove(JOB_FOCUS_END); - mOrchestrator.remove(JOB_FOCUS_RESET); - mCallback.dispatchOnFocusEnd(gesture, success, p); + getOrchestrator().remove(JOB_FOCUS_END); + getOrchestrator().remove(JOB_FOCUS_RESET); + getCallback().dispatchOnFocusEnd(gesture, success, p); if (shouldResetAutoFocus()) { - mOrchestrator.scheduleStatefulDelayed( + getOrchestrator().scheduleStatefulDelayed( JOB_FOCUS_RESET, CameraState.ENGINE, getAutoFocusResetDelay(), diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java index f3a59228..bf64da89 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java @@ -34,7 +34,6 @@ import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.TaskCompletionSource; import com.google.android.gms.tasks.Tasks; import com.otaliastudios.cameraview.CameraException; -import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.CameraOptions; import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.VideoResult; @@ -59,16 +58,15 @@ import com.otaliastudios.cameraview.engine.options.Camera2Options; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.frame.FrameManager; +import com.otaliastudios.cameraview.frame.ImageFrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.internal.utils.CropHelper; -import com.otaliastudios.cameraview.internal.utils.ImageHelper; import com.otaliastudios.cameraview.internal.utils.WorkerHandler; import com.otaliastudios.cameraview.picture.Full2PictureRecorder; import com.otaliastudios.cameraview.picture.Snapshot2PictureRecorder; import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.Size; -import com.otaliastudios.cameraview.size.SizeSelectors; import com.otaliastudios.cameraview.video.Full2VideoRecorder; import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; @@ -79,10 +77,12 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; @RequiresApi(Build.VERSION_CODES.LOLLIPOP) -public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAvailableListener, +public class Camera2Engine extends CameraBaseEngine implements + ImageReader.OnImageAvailableListener, ActionHolder { - private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21; - private static final int FRAME_PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888; + + private static final int FRAME_PROCESSING_POOL_SIZE = 2; + private static final int FRAME_PROCESSING_FORMAT = ImageFormat.YUV_420_888; @VisibleForTesting static final long METER_TIMEOUT = 2500; private final CameraManager mManager; @@ -95,10 +95,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv private final Camera2Mapper mMapper = Camera2Mapper.get(); // Frame processing - private Size mFrameProcessingSize; private ImageReader mFrameProcessingReader; // need this or the reader surface is collected - private final WorkerHandler mFrameConversionHandler; - private final Object mFrameProcessingImageLock = new Object(); private Surface mFrameProcessingSurface; // Preview @@ -119,8 +116,8 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public Camera2Engine(Callback callback) { super(callback); - mManager = (CameraManager) mCallback.getContext().getSystemService(Context.CAMERA_SERVICE); - mFrameConversionHandler = WorkerHandler.get("CameraFrameConversion"); + mManager = (CameraManager) getCallback().getContext() + .getSystemService(Context.CAMERA_SERVICE); new LogAction().start(this); } @@ -326,6 +323,29 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv } } + @EngineThread + @NonNull + @Override + protected List getFrameProcessingAvailableSizes() { + try { + CameraCharacteristics characteristics = mManager.getCameraCharacteristics(mCameraId); + StreamConfigurationMap streamMap = + characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + if (streamMap == null) { + throw new RuntimeException("StreamConfigurationMap is null. Should not happen."); + } + android.util.Size[] sizes = streamMap.getOutputSizes(mFrameProcessingFormat); + List candidates = new ArrayList<>(sizes.length); + for (android.util.Size size : sizes) { + Size add = new Size(size.getWidth(), size.getHeight()); + if (!candidates.contains(add)) candidates.add(add); + } + return candidates; + } catch (CameraAccessException e) { + throw createCameraException(e); + } + } + @EngineThread @Override protected void onPreviewStreamSizeChanged() { @@ -520,28 +540,13 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv // 4. FRAME PROCESSING if (hasFrameProcessors()) { - // Choose the size. - StreamConfigurationMap streamMap = mCameraCharacteristics - .get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - if (streamMap == null) { - throw new RuntimeException("StreamConfigurationMap is null. Should not happen."); - } - android.util.Size[] aSizes = streamMap.getOutputSizes(FRAME_PROCESSING_INPUT_FORMAT); - List sizes = new ArrayList<>(); - for (android.util.Size aSize : aSizes) { - sizes.add(new Size(aSize.getWidth(), aSize.getHeight())); - } - mFrameProcessingSize = SizeSelectors.and( - SizeSelectors.maxWidth(Math.min(640, mPreviewStreamSize.getWidth())), - SizeSelectors.maxHeight(Math.min(640, mPreviewStreamSize.getHeight())), - SizeSelectors.biggest()).select(sizes).get(0); + mFrameProcessingSize = computeFrameProcessingSize(); mFrameProcessingReader = ImageReader.newInstance( mFrameProcessingSize.getWidth(), mFrameProcessingSize.getHeight(), - FRAME_PROCESSING_INPUT_FORMAT, - 2); - mFrameProcessingReader.setOnImageAvailableListener(this, - mFrameConversionHandler.getHandler()); + mFrameProcessingFormat, + getFrameProcessingPoolSize()); + mFrameProcessingReader.setOnImageAvailableListener(this, null); mFrameProcessingSurface = mFrameProcessingReader.getSurface(); outputSurfaces.add(mFrameProcessingSurface); } else { @@ -584,7 +589,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv @Override protected Task onStartPreview() { LOG.i("onStartPreview:", "Dispatching onCameraPreviewStreamSizeChanged."); - mCallback.onCameraPreviewStreamSizeChanged(); + getCallback().onCameraPreviewStreamSizeChanged(); Size previewSizeForView = getPreviewStreamSize(Reference.VIEW); if (previewSizeForView == null) { @@ -593,7 +598,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv mPreview.setStreamSize(previewSizeForView.getWidth(), previewSizeForView.getHeight()); mPreview.setDrawRotation(getAngles().offset(Reference.BASE, Reference.VIEW, Axis.ABSOLUTE)); if (hasFrameProcessors()) { - getFrameManager().setUp(FRAME_PROCESSING_FORMAT, mFrameProcessingSize); + getFrameManager().setUp(mFrameProcessingFormat, mFrameProcessingSize); } LOG.i("onStartPreview:", "Starting preview."); @@ -608,7 +613,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv // the recorder sets. Also we are posting so that doTakeVideo sees a started preview. final VideoResult.Stub stub = mFullVideoPendingStub; mFullVideoPendingStub = null; - mOrchestrator.scheduleStateful("do take video", CameraState.PREVIEW, + getOrchestrator().scheduleStateful("do take video", CameraState.PREVIEW, new Runnable() { @Override public void run() { @@ -675,12 +680,9 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv mCaptureSize = null; mFrameProcessingSize = null; if (mFrameProcessingReader != null) { - synchronized (mFrameProcessingImageLock) { - // This call synchronously releases all Images and their underlying properties. - // This can cause a segmentation fault while converting the Image to NV21. - // So we use this lock for the two operations. - mFrameProcessingReader.close(); - } + // WARNING: This call synchronously releases all Images and their underlying + // properties. This can cause issues if the Image is being used. + mFrameProcessingReader.close(); mFrameProcessingReader = null; } if (mPictureReader != null) { @@ -829,7 +831,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv boolean unlock = (fullPicture && getPictureMetering()) || (!fullPicture && getPictureSnapshotMetering()); if (unlock) { - mOrchestrator.scheduleStateful("reset metering after picture", + getOrchestrator().scheduleStateful("reset metering after picture", CameraState.PREVIEW, new Runnable() { @Override @@ -945,7 +947,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv // SnapshotRecorder will invoke this on its own thread, so let's post in our own thread // and check camera state before trying to restore the preview. Engine might have been // torn down in the engine thread while this was still being called. - mOrchestrator.scheduleStateful("restore preview template", CameraState.BIND, + getOrchestrator().scheduleStateful("restore preview template", CameraState.BIND, new Runnable() { @Override public void run() { @@ -1078,7 +1080,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setFlash(@NonNull final Flash flash) { final Flash old = mFlash; mFlash = flash; - mFlashTask = mOrchestrator.scheduleStateful("flash (" + flash + ")", + mFlashTask = getOrchestrator().scheduleStateful("flash (" + flash + ")", CameraState.ENGINE, new Runnable() { @Override @@ -1153,7 +1155,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setLocation(@Nullable Location location) { final Location old = mLocation; mLocation = location; - mLocationTask = mOrchestrator.scheduleStateful("location", + mLocationTask = getOrchestrator().scheduleStateful("location", CameraState.ENGINE, new Runnable() { @Override @@ -1178,7 +1180,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setWhiteBalance(@NonNull WhiteBalance whiteBalance) { final WhiteBalance old = mWhiteBalance; mWhiteBalance = whiteBalance; - mWhiteBalanceTask = mOrchestrator.scheduleStateful( + mWhiteBalanceTask = getOrchestrator().scheduleStateful( "white balance (" + whiteBalance + ")", CameraState.ENGINE, new Runnable() { @@ -1207,7 +1209,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setHdr(@NonNull Hdr hdr) { final Hdr old = mHdr; mHdr = hdr; - mHdrTask = mOrchestrator.scheduleStateful("hdr (" + hdr + ")", + mHdrTask = getOrchestrator().scheduleStateful("hdr (" + hdr + ")", CameraState.ENGINE, new Runnable() { @Override @@ -1234,7 +1236,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setZoom(final float zoom, final @Nullable PointF[] points, final boolean notify) { final float old = mZoomValue; mZoomValue = zoom; - mZoomTask = mOrchestrator.scheduleStateful( + mZoomTask = getOrchestrator().scheduleStateful( "zoom (" + zoom + ")", CameraState.ENGINE, new Runnable() { @@ -1243,7 +1245,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (applyZoom(mRepeatingRequestBuilder, old)) { applyRepeatingRequestBuilder(); if (notify) { - mCallback.dispatchOnZoomChanged(zoom, points); + getCallback().dispatchOnZoomChanged(zoom, points); } } } @@ -1290,7 +1292,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv final boolean notify) { final float old = mExposureCorrectionValue; mExposureCorrectionValue = EVvalue; - mExposureCorrectionTask = mOrchestrator.scheduleStateful( + mExposureCorrectionTask = getOrchestrator().scheduleStateful( "exposure correction (" + EVvalue + ")", CameraState.ENGINE, new Runnable() { @@ -1299,7 +1301,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (applyExposureCorrection(mRepeatingRequestBuilder, old)) { applyRepeatingRequestBuilder(); if (notify) { - mCallback.dispatchOnExposureCorrectionChanged(EVvalue, bounds, points); + getCallback().dispatchOnExposureCorrectionChanged(EVvalue, bounds, points); } } } @@ -1332,7 +1334,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setPreviewFrameRate(float previewFrameRate) { final float oldPreviewFrameRate = mPreviewFrameRate; mPreviewFrameRate = previewFrameRate; - mPreviewFrameRateTask = mOrchestrator.scheduleStateful( + mPreviewFrameRateTask = getOrchestrator().scheduleStateful( "preview fps (" + previewFrameRate + ")", CameraState.ENGINE, new Runnable() { @@ -1382,7 +1384,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setPictureFormat(final @NonNull PictureFormat pictureFormat) { if (pictureFormat != mPictureFormat) { mPictureFormat = pictureFormat; - mOrchestrator.scheduleStateful("picture format (" + pictureFormat + ")", + getOrchestrator().scheduleStateful("picture format (" + pictureFormat + ")", CameraState.ENGINE, new Runnable() { @Override @@ -1397,50 +1399,36 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv //region Frame Processing + protected int getFrameProcessingPoolSize() { + return FRAME_PROCESSING_POOL_SIZE; + } + @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, null); + return new ImageFrameManager(getFrameProcessingPoolSize()); } @Override public void onImageAvailable(ImageReader reader) { - byte[] data = getFrameManager().getBuffer(); - if (data == null) { - LOG.w("onImageAvailable", "no byte buffer!"); - return; - } LOG.v("onImageAvailable", "trying to acquire Image."); Image image = null; try { image = reader.acquireLatestImage(); } catch (Exception ignore) { } if (image == null) { - LOG.w("onImageAvailable", "we have a byte buffer but no Image!"); - getFrameManager().onBufferUnused(data); - return; - } - LOG.v("onImageAvailable", "we have both a byte buffer and an Image."); - try { - synchronized (mFrameProcessingImageLock) { - ImageHelper.convertToNV21(image, data); - } - } catch (Exception e) { - LOG.w("onImageAvailable", "error while converting."); - getFrameManager().onBufferUnused(data); - image.close(); - return; - } - image.close(); - if (getState() == CameraState.PREVIEW && !isChangingState()) { + LOG.w("onImageAvailable", "failed to acquire Image!"); + } else if (getState() == CameraState.PREVIEW && !isChangingState()) { // After preview, the frame manager is correctly set up - Frame frame = getFrameManager().getFrame(data, + //noinspection unchecked + Frame frame = getFrameManager().getFrame(image, System.currentTimeMillis(), - getAngles().offset(Reference.SENSOR, Reference.OUTPUT, + getAngles().offset(Reference.SENSOR, + Reference.OUTPUT, Axis.RELATIVE_TO_SENSOR)); - mCallback.dispatchFrame(frame); + getCallback().dispatchFrame(frame); } else { - getFrameManager().onBufferUnused(data); + image.close(); } } @@ -1448,7 +1436,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv public void setHasFrameProcessors(final boolean hasFrameProcessors) { // Frame processing is set up partially when binding and partially when starting // the preview. If the value is changed between the two, the preview step can crash. - mOrchestrator.schedule("has frame processors (" + hasFrameProcessors + ")", + getOrchestrator().schedule("has frame processors (" + hasFrameProcessors + ")", true, new Runnable() { @Override public void run() { @@ -1456,13 +1444,36 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv // Extremely rare case in which this was called in between startBind and // startPreview. This can cause issues. Try later. setHasFrameProcessors(hasFrameProcessors); - } else if (getState().isAtLeast(CameraState.BIND)) { - // Apply and restart. - Camera2Engine.super.setHasFrameProcessors(hasFrameProcessors); + return; + } + // Apply and restart. + mHasFrameProcessors = hasFrameProcessors; + if (getState().isAtLeast(CameraState.BIND)) { + restartBind(); + } + } + }); + } + + @Override + public void setFrameProcessingFormat(final int format) { + // This is called during initialization. Set our default first. + if (mFrameProcessingFormat == 0) mFrameProcessingFormat = FRAME_PROCESSING_FORMAT; + // Frame processing format is used both when binding and when starting the preview. + // If the value is changed between the two, the preview step can crash. + getOrchestrator().schedule("frame processing format (" + format + ")", + true, new Runnable() { + @Override + public void run() { + if (getState().isAtLeast(CameraState.BIND) && isChangingState()) { + // Extremely rare case in which this was called in between startBind and + // startPreview. This can cause issues. Try later. + setFrameProcessingFormat(format); + return; + } + mFrameProcessingFormat = format > 0 ? format : FRAME_PROCESSING_FORMAT; + if (getState().isAtLeast(CameraState.BIND)) { restartBind(); - } else { - // Just apply. - Camera2Engine.super.setHasFrameProcessors(hasFrameProcessors); } } }); @@ -1477,7 +1488,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv // This will only work when we have a preview, since it launches the preview // in the end. Even without this it would need the bind state at least, // since we need the preview size. - mOrchestrator.scheduleStateful("autofocus (" + gesture + ")", + getOrchestrator().scheduleStateful("autofocus (" + gesture + ")", CameraState.PREVIEW, new Runnable() { @Override @@ -1487,17 +1498,17 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (!mCameraOptions.isAutoFocusSupported()) return; // Create the meter and start. - mCallback.dispatchOnFocusStart(gesture, point); + getCallback().dispatchOnFocusStart(gesture, point); final MeterAction action = createMeterAction(point); Action wrapper = Actions.timeout(METER_TIMEOUT, action); wrapper.start(Camera2Engine.this); wrapper.addCallback(new CompletionCallback() { @Override protected void onActionCompleted(@NonNull Action a) { - mCallback.dispatchOnFocusEnd(gesture, action.isSuccessful(), point); - mOrchestrator.remove("reset metering"); + getCallback().dispatchOnFocusEnd(gesture, action.isSuccessful(), point); + getOrchestrator().remove("reset metering"); if (shouldResetAutoFocus()) { - mOrchestrator.scheduleStatefulDelayed("reset metering", + getOrchestrator().scheduleStatefulDelayed("reset metering", CameraState.PREVIEW, getAutoFocusResetDelay(), new Runnable() { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java new file mode 100644 index 00000000..e2cd5ae9 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java @@ -0,0 +1,927 @@ +package com.otaliastudios.cameraview.engine; + +import android.location.Location; + +import androidx.annotation.CallSuper; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import com.google.android.gms.tasks.Task; +import com.google.android.gms.tasks.Tasks; +import com.otaliastudios.cameraview.CameraException; +import com.otaliastudios.cameraview.CameraOptions; +import com.otaliastudios.cameraview.PictureResult; +import com.otaliastudios.cameraview.VideoResult; +import com.otaliastudios.cameraview.controls.Audio; +import com.otaliastudios.cameraview.controls.Facing; +import com.otaliastudios.cameraview.controls.Flash; +import com.otaliastudios.cameraview.controls.Hdr; +import com.otaliastudios.cameraview.controls.Mode; +import com.otaliastudios.cameraview.controls.PictureFormat; +import com.otaliastudios.cameraview.controls.VideoCodec; +import com.otaliastudios.cameraview.controls.WhiteBalance; +import com.otaliastudios.cameraview.engine.offset.Angles; +import com.otaliastudios.cameraview.engine.offset.Reference; +import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.FrameManager; +import com.otaliastudios.cameraview.overlay.Overlay; +import com.otaliastudios.cameraview.picture.PictureRecorder; +import com.otaliastudios.cameraview.preview.CameraPreview; +import com.otaliastudios.cameraview.size.AspectRatio; +import com.otaliastudios.cameraview.size.Size; +import com.otaliastudios.cameraview.size.SizeSelector; +import com.otaliastudios.cameraview.size.SizeSelectors; +import com.otaliastudios.cameraview.video.VideoRecorder; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +/** + * Abstract implementation of {@link CameraEngine} that helps in common tasks. + */ +public abstract class CameraBaseEngine extends CameraEngine { + + @SuppressWarnings("WeakerAccess") protected CameraPreview mPreview; + @SuppressWarnings("WeakerAccess") protected CameraOptions mCameraOptions; + @SuppressWarnings("WeakerAccess") protected PictureRecorder mPictureRecorder; + @SuppressWarnings("WeakerAccess") protected VideoRecorder mVideoRecorder; + @SuppressWarnings("WeakerAccess") protected Size mCaptureSize; + @SuppressWarnings("WeakerAccess") protected Size mPreviewStreamSize; + @SuppressWarnings("WeakerAccess") protected Size mFrameProcessingSize; + @SuppressWarnings("WeakerAccess") protected int mFrameProcessingFormat; + @SuppressWarnings("WeakerAccess") protected boolean mHasFrameProcessors; + @SuppressWarnings("WeakerAccess") protected Flash mFlash; + @SuppressWarnings("WeakerAccess") protected WhiteBalance mWhiteBalance; + @SuppressWarnings("WeakerAccess") protected VideoCodec mVideoCodec; + @SuppressWarnings("WeakerAccess") protected Hdr mHdr; + @SuppressWarnings("WeakerAccess") protected PictureFormat mPictureFormat; + @SuppressWarnings("WeakerAccess") protected Location mLocation; + @SuppressWarnings("WeakerAccess") protected float mZoomValue; + @SuppressWarnings("WeakerAccess") protected float mExposureCorrectionValue; + @SuppressWarnings("WeakerAccess") protected boolean mPlaySounds; + @SuppressWarnings("WeakerAccess") protected boolean mPictureMetering; + @SuppressWarnings("WeakerAccess") protected boolean mPictureSnapshotMetering; + @SuppressWarnings("WeakerAccess") protected float mPreviewFrameRate; + + private final FrameManager mFrameManager; + private final Angles mAngles; + @Nullable private SizeSelector mPreviewStreamSizeSelector; + private SizeSelector mPictureSizeSelector; + private SizeSelector mVideoSizeSelector; + private Facing mFacing; + private Mode mMode; + private Audio mAudio; + private long mVideoMaxSize; + private int mVideoMaxDuration; + private int mVideoBitRate; + private int mAudioBitRate; + private long mAutoFocusResetDelayMillis; + private int mSnapshotMaxWidth; // in REF_VIEW like SizeSelectors + private int mSnapshotMaxHeight; // in REF_VIEW like SizeSelectors + private int mFrameProcessingMaxWidth; // in REF_VIEW like SizeSelectors + private int mFrameProcessingMaxHeight; // in REF_VIEW like SizeSelectors + private Overlay mOverlay; + + // Ops used for testing. + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mZoomTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mExposureCorrectionTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mFlashTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mWhiteBalanceTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mHdrTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mLocationTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mPlaySoundsTask + = Tasks.forResult(null); + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mPreviewFrameRateTask + = Tasks.forResult(null); + + @SuppressWarnings("WeakerAccess") + protected CameraBaseEngine(@NonNull Callback callback) { + super(callback); + mFrameManager = instantiateFrameManager(); + mAngles = new Angles(); + } + + /** + * Called at construction time to get a frame manager that can later be + * accessed through {@link #getFrameManager()}. + * @return a frame manager + */ + @NonNull + protected abstract FrameManager instantiateFrameManager(); + + @NonNull + @Override + public final Angles getAngles() { + return mAngles; + } + + @NonNull + @Override + public FrameManager getFrameManager() { + return mFrameManager; + } + + @Nullable + @Override + public final CameraOptions getCameraOptions() { + return mCameraOptions; + } + + @Override + public final void setPreview(@NonNull CameraPreview cameraPreview) { + if (mPreview != null) mPreview.setSurfaceCallback(null); + mPreview = cameraPreview; + mPreview.setSurfaceCallback(this); + } + + @NonNull + @Override + public final CameraPreview getPreview() { + return mPreview; + } + + @Override + public final void setOverlay(@Nullable Overlay overlay) { + mOverlay = overlay; + } + + @Nullable + @Override + public final Overlay getOverlay() { + return mOverlay; + } + + @Override + public final void setPreviewStreamSizeSelector(@Nullable SizeSelector selector) { + mPreviewStreamSizeSelector = selector; + } + + @Nullable + @Override + public final SizeSelector getPreviewStreamSizeSelector() { + return mPreviewStreamSizeSelector; + } + + @Override + public final void setPictureSizeSelector(@NonNull SizeSelector selector) { + mPictureSizeSelector = selector; + } + + @NonNull + @Override + public final SizeSelector getPictureSizeSelector() { + return mPictureSizeSelector; + } + + @Override + public final void setVideoSizeSelector(@NonNull SizeSelector selector) { + mVideoSizeSelector = selector; + } + + @NonNull + @Override + public final SizeSelector getVideoSizeSelector() { + return mVideoSizeSelector; + } + + @Override + public final void setVideoMaxSize(long videoMaxSizeBytes) { + mVideoMaxSize = videoMaxSizeBytes; + } + + @Override + public final long getVideoMaxSize() { + return mVideoMaxSize; + } + + @Override + public final void setVideoMaxDuration(int videoMaxDurationMillis) { + mVideoMaxDuration = videoMaxDurationMillis; + } + + @Override + public final int getVideoMaxDuration() { + return mVideoMaxDuration; + } + + @Override + public final void setVideoCodec(@NonNull VideoCodec codec) { + mVideoCodec = codec; + } + + @NonNull + @Override + public final VideoCodec getVideoCodec() { + return mVideoCodec; + } + + @Override + public final void setVideoBitRate(int videoBitRate) { + mVideoBitRate = videoBitRate; + } + + @Override + public final int getVideoBitRate() { + return mVideoBitRate; + } + + @Override + public final void setAudioBitRate(int audioBitRate) { + mAudioBitRate = audioBitRate; + } + + @Override + public final int getAudioBitRate() { + return mAudioBitRate; + } + + @Override + public final void setSnapshotMaxWidth(int maxWidth) { + mSnapshotMaxWidth = maxWidth; + } + + @Override + public final int getSnapshotMaxWidth() { + return mSnapshotMaxWidth; + } + + @Override + public final void setSnapshotMaxHeight(int maxHeight) { + mSnapshotMaxHeight = maxHeight; + } + + @Override + public final int getSnapshotMaxHeight() { + return mSnapshotMaxHeight; + } + + @Override + public final void setFrameProcessingMaxWidth(int maxWidth) { + mFrameProcessingMaxWidth = maxWidth; + } + + @Override + public final int getFrameProcessingMaxWidth() { + return mFrameProcessingMaxWidth; + } + + @Override + public final void setFrameProcessingMaxHeight(int maxHeight) { + mFrameProcessingMaxHeight = maxHeight; + } + + @Override + public final int getFrameProcessingMaxHeight() { + return mFrameProcessingMaxHeight; + } + + @Override + public final int getFrameProcessingFormat() { + return mFrameProcessingFormat; + } + + @Override + public final void setAutoFocusResetDelay(long delayMillis) { + mAutoFocusResetDelayMillis = delayMillis; + } + + @Override + public final long getAutoFocusResetDelay() { + return mAutoFocusResetDelayMillis; + } + + /** + * Helper function for subclasses. + * @return true if AF should be reset + */ + @SuppressWarnings("WeakerAccess") + protected final boolean shouldResetAutoFocus() { + return mAutoFocusResetDelayMillis > 0 && mAutoFocusResetDelayMillis != Long.MAX_VALUE; + } + + /** + * Sets a new facing value. This will restart the engine session (if there's any) + * so that we can open the new facing camera. + * @param facing facing + */ + @Override + public final void setFacing(final @NonNull Facing facing) { + final Facing old = mFacing; + if (facing != old) { + mFacing = facing; + getOrchestrator().scheduleStateful("facing", CameraState.ENGINE, + new Runnable() { + @Override + public void run() { + if (collectCameraInfo(facing)) { + restart(); + } else { + mFacing = old; + } + } + }); + } + } + + @NonNull + @Override + public final Facing getFacing() { + return mFacing; + } + + /** + * Sets a new audio value that will be used for video recordings. + * @param audio desired audio + */ + @Override + public final void setAudio(@NonNull Audio audio) { + if (mAudio != audio) { + if (isTakingVideo()) { + LOG.w("Audio setting was changed while recording. " + + "Changes will take place starting from next video"); + } + mAudio = audio; + } + } + + @NonNull + @Override + public final Audio getAudio() { + return mAudio; + } + + /** + * Sets the desired mode (either picture or video). + * @param mode desired mode. + */ + @Override + public final void setMode(@NonNull Mode mode) { + if (mode != mMode) { + mMode = mode; + getOrchestrator().scheduleStateful("mode", CameraState.ENGINE, + new Runnable() { + @Override + public void run() { + restart(); + } + }); + } + } + + @NonNull + @Override + public final Mode getMode() { + return mMode; + } + + @Override + public final float getZoomValue() { + return mZoomValue; + } + + @Override + public final float getExposureCorrectionValue() { + return mExposureCorrectionValue; + } + + @NonNull + @Override + public final Flash getFlash() { + return mFlash; + } + + @NonNull + @Override + public final WhiteBalance getWhiteBalance() { + return mWhiteBalance; + } + + @NonNull + @Override + public final Hdr getHdr() { + return mHdr; + } + + @Nullable + @Override + public final Location getLocation() { + return mLocation; + } + + @NonNull + @Override + public final PictureFormat getPictureFormat() { + return mPictureFormat; + } + + @Override + public final float getPreviewFrameRate() { + return mPreviewFrameRate; + } + + @Override + public final boolean hasFrameProcessors() { + return mHasFrameProcessors; + } + + @Override + public final void setPictureMetering(boolean enable) { + mPictureMetering = enable; + } + + @Override + public final boolean getPictureMetering() { + return mPictureMetering; + } + + @Override + public final void setPictureSnapshotMetering(boolean enable) { + mPictureSnapshotMetering = enable; + } + + @Override + public final boolean getPictureSnapshotMetering() { + return mPictureSnapshotMetering; + } + + //region Picture and video control + + @Override + public final boolean isTakingPicture() { + return mPictureRecorder != null; + } + + @Override + public /* final */ void takePicture(final @NonNull PictureResult.Stub stub) { + // Save boolean before scheduling! See how Camera2Engine calls this with a temp value. + final boolean metering = mPictureMetering; + getOrchestrator().scheduleStateful("take picture", CameraState.BIND, + new Runnable() { + @Override + public void run() { + LOG.i("takePicture:", "running. isTakingPicture:", isTakingPicture()); + if (isTakingPicture()) return; + if (mMode == Mode.VIDEO) { + throw new IllegalStateException("Can't take hq pictures while in VIDEO mode"); + } + stub.isSnapshot = false; + stub.location = mLocation; + stub.facing = mFacing; + stub.format = mPictureFormat; + onTakePicture(stub, metering); + } + }); + } + + /** + * The snapshot size is the {@link #getPreviewStreamSize(Reference)}, but cropped based on the + * view/surface aspect ratio. + * @param stub a picture stub + */ + @Override + public /* final */ void takePictureSnapshot(final @NonNull PictureResult.Stub stub) { + // Save boolean before scheduling! See how Camera2Engine calls this with a temp value. + final boolean metering = mPictureSnapshotMetering; + getOrchestrator().scheduleStateful("take picture snapshot", CameraState.BIND, + new Runnable() { + @Override + public void run() { + LOG.i("takePictureSnapshot:", "running. isTakingPicture:", isTakingPicture()); + if (isTakingPicture()) return; + stub.location = mLocation; + stub.isSnapshot = true; + stub.facing = mFacing; + stub.format = PictureFormat.JPEG; + // Leave the other parameters to subclasses. + //noinspection ConstantConditions + AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); + onTakePictureSnapshot(stub, ratio, metering); + } + }); + } + + @Override + public void onPictureShutter(boolean didPlaySound) { + getCallback().onShutter(!didPlaySound); + } + + @Override + public void onPictureResult(@Nullable PictureResult.Stub result, @Nullable Exception error) { + mPictureRecorder = null; + if (result != null) { + getCallback().dispatchOnPictureTaken(result); + } else { + LOG.e("onPictureResult", "result is null: something went wrong.", error); + getCallback().dispatchError(new CameraException(error, + CameraException.REASON_PICTURE_FAILED)); + } + } + + @Override + public final boolean isTakingVideo() { + return mVideoRecorder != null && mVideoRecorder.isRecording(); + } + + @Override + public final void takeVideo(final @NonNull VideoResult.Stub stub, final @NonNull File file) { + getOrchestrator().scheduleStateful("take video", CameraState.BIND, new Runnable() { + @Override + public void run() { + LOG.i("takeVideo:", "running. isTakingVideo:", isTakingVideo()); + if (isTakingVideo()) return; + if (mMode == Mode.PICTURE) { + throw new IllegalStateException("Can't record video while in PICTURE mode"); + } + stub.file = file; + stub.isSnapshot = false; + stub.videoCodec = mVideoCodec; + stub.location = mLocation; + stub.facing = mFacing; + stub.audio = mAudio; + stub.maxSize = mVideoMaxSize; + stub.maxDuration = mVideoMaxDuration; + stub.videoBitRate = mVideoBitRate; + stub.audioBitRate = mAudioBitRate; + onTakeVideo(stub); + } + }); + } + + /** + * @param stub a video stub + * @param file the output file + */ + @Override + public final void takeVideoSnapshot(@NonNull final VideoResult.Stub stub, + @NonNull final File file) { + getOrchestrator().scheduleStateful("take video snapshot", CameraState.BIND, + new Runnable() { + @Override + public void run() { + LOG.i("takeVideoSnapshot:", "running. isTakingVideo:", isTakingVideo()); + stub.file = file; + stub.isSnapshot = true; + stub.videoCodec = mVideoCodec; + stub.location = mLocation; + stub.facing = mFacing; + stub.videoBitRate = mVideoBitRate; + stub.audioBitRate = mAudioBitRate; + stub.audio = mAudio; + stub.maxSize = mVideoMaxSize; + stub.maxDuration = mVideoMaxDuration; + //noinspection ConstantConditions + AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); + onTakeVideoSnapshot(stub, ratio); + } + }); + } + + @Override + public final void stopVideo() { + getOrchestrator().schedule("stop video", true, new Runnable() { + @Override + public void run() { + LOG.i("stopVideo", "running. isTakingVideo?", isTakingVideo()); + onStopVideo(); + } + }); + } + + @EngineThread + @SuppressWarnings("WeakerAccess") + protected void onStopVideo() { + if (mVideoRecorder != null) { + mVideoRecorder.stop(false); + // Do not null this, so we respond correctly to isTakingVideo(), + // which checks for recorder presence and recorder.isRecording(). + // It will be nulled in onVideoResult. + } + } + + @CallSuper + @Override + public void onVideoResult(@Nullable VideoResult.Stub result, @Nullable Exception exception) { + mVideoRecorder = null; + if (result != null) { + getCallback().dispatchOnVideoTaken(result); + } else { + LOG.e("onVideoResult", "result is null: something went wrong.", exception); + getCallback().dispatchError(new CameraException(exception, + CameraException.REASON_VIDEO_FAILED)); + } + } + + @Override + public void onVideoRecordingStart() { + getCallback().dispatchOnVideoRecordingStart(); + } + + @Override + public void onVideoRecordingEnd() { + getCallback().dispatchOnVideoRecordingEnd(); + } + + @EngineThread + protected abstract void onTakePicture(@NonNull PictureResult.Stub stub, boolean doMetering); + + @EngineThread + protected abstract void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, + @NonNull AspectRatio outputRatio, + boolean doMetering); + + @EngineThread + protected abstract void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, + @NonNull AspectRatio outputRatio); + + @EngineThread + protected abstract void onTakeVideo(@NonNull VideoResult.Stub stub); + + //endregion + + //region Size / Surface + + @Override + public final void onSurfaceChanged() { + LOG.i("onSurfaceChanged:", "Size is", getPreviewSurfaceSize(Reference.VIEW)); + getOrchestrator().scheduleStateful("surface changed", CameraState.BIND, + new Runnable() { + @Override + public void run() { + // Compute a new camera preview size and apply. + Size newSize = computePreviewStreamSize(); + if (newSize.equals(mPreviewStreamSize)) { + LOG.i("onSurfaceChanged:", + "The computed preview size is identical. No op."); + } else { + LOG.i("onSurfaceChanged:", + "Computed a new preview size. Calling onPreviewStreamSizeChanged()."); + mPreviewStreamSize = newSize; + onPreviewStreamSizeChanged(); + } + } + }); + } + + /** + * The preview stream size has changed. At this point, some engine might want to + * simply call {@link #restartPreview()}, others to {@link #restartBind()}. + * + * It basically depends on the step at which the preview stream size is actually used. + */ + @EngineThread + protected abstract void onPreviewStreamSizeChanged(); + + @Nullable + @Override + public final Size getPictureSize(@SuppressWarnings("SameParameterValue") @NonNull Reference reference) { + Size size = mCaptureSize; + if (size == null || mMode == Mode.VIDEO) return null; + return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; + } + + @Nullable + @Override + public final Size getVideoSize(@SuppressWarnings("SameParameterValue") @NonNull Reference reference) { + Size size = mCaptureSize; + if (size == null || mMode == Mode.PICTURE) return null; + return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; + } + + @Nullable + @Override + public final Size getPreviewStreamSize(@NonNull Reference reference) { + Size size = mPreviewStreamSize; + if (size == null) return null; + return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; + } + + @SuppressWarnings("SameParameterValue") + @Nullable + private Size getPreviewSurfaceSize(@NonNull Reference reference) { + CameraPreview preview = mPreview; + if (preview == null) return null; + return getAngles().flip(Reference.VIEW, reference) ? preview.getSurfaceSize().flip() + : preview.getSurfaceSize(); + } + + /** + * Returns the snapshot size, but not cropped with the view dimensions, which + * is what we will do before creating the snapshot. However, cropping is done at various + * levels so we don't want to perform the op here. + * + * The base snapshot size is based on PreviewStreamSize (later cropped with view ratio). Why? + * One might be tempted to say that it's the SurfaceSize (which already matches the view ratio). + * + * The camera sensor will capture preview frames with PreviewStreamSize and that's it. Then they + * are hardware-scaled by the preview surface, but this does not affect the snapshot, as the + * snapshot recorder simply creates another surface. + * + * Done tests to ensure that this is true, by using + * 1. small SurfaceSize and biggest() PreviewStreamSize: output is not low quality + * 2. big SurfaceSize and smallest() PreviewStreamSize: output is low quality + * In both cases the result.size here was set to the biggest of the two. + * + * I could not find the same evidence for videos, but I would say that the same things should + * apply, despite the capturing mechanism being different. + * + * @param reference the reference system + * @return the uncropped snapshot size + */ + @Nullable + @Override + public final Size getUncroppedSnapshotSize(@NonNull Reference reference) { + Size baseSize = getPreviewStreamSize(reference); + if (baseSize == null) return null; + boolean flip = getAngles().flip(reference, Reference.VIEW); + int maxWidth = flip ? mSnapshotMaxHeight : mSnapshotMaxWidth; + int maxHeight = flip ? mSnapshotMaxWidth : mSnapshotMaxHeight; + if (maxWidth <= 0) maxWidth = Integer.MAX_VALUE; + if (maxHeight <= 0) maxHeight = Integer.MAX_VALUE; + float baseRatio = AspectRatio.of(baseSize).toFloat(); + float maxValuesRatio = AspectRatio.of(maxWidth, maxHeight).toFloat(); + if (maxValuesRatio >= baseRatio) { + // Height is the real constraint. + int outHeight = Math.min(baseSize.getHeight(), maxHeight); + int outWidth = (int) Math.floor((float) outHeight * baseRatio); + return new Size(outWidth, outHeight); + } else { + // Width is the real constraint. + int outWidth = Math.min(baseSize.getWidth(), maxWidth); + int outHeight = (int) Math.floor((float) outWidth / baseRatio); + return new Size(outWidth, outHeight); + } + } + + /** + * This is called either on cameraView.start(), or when the underlying surface changes. + * It is possible that in the first call the preview surface has not already computed its + * dimensions. + * But when it does, the {@link CameraPreview.SurfaceCallback} should be called, + * and this should be refreshed. + * + * @return the capture size + */ + @NonNull + @SuppressWarnings("WeakerAccess") + protected final Size computeCaptureSize() { + return computeCaptureSize(mMode); + } + + @NonNull + @SuppressWarnings("WeakerAccess") + protected final Size computeCaptureSize(@NonNull Mode mode) { + // We want to pass stuff into the REF_VIEW reference, not the sensor one. + // This is already managed by CameraOptions, so we just flip again at the end. + boolean flip = getAngles().flip(Reference.SENSOR, Reference.VIEW); + SizeSelector selector; + Collection sizes; + if (mode == Mode.PICTURE) { + selector = mPictureSizeSelector; + sizes = mCameraOptions.getSupportedPictureSizes(); + } else { + selector = mVideoSizeSelector; + sizes = mCameraOptions.getSupportedVideoSizes(); + } + selector = SizeSelectors.or(selector, SizeSelectors.biggest()); + List list = new ArrayList<>(sizes); + Size result = selector.select(list).get(0); + if (!list.contains(result)) { + throw new RuntimeException("SizeSelectors must not return Sizes other than " + + "those in the input list."); + } + LOG.i("computeCaptureSize:", "result:", result, "flip:", flip, "mode:", mode); + if (flip) result = result.flip(); // Go back to REF_SENSOR + return result; + } + + /** + * This is called anytime {@link #computePreviewStreamSize()} is called. + * This means that it should be called during the binding process, when + * we can be sure that the camera is available (engineState == STARTED). + * @return a list of available sizes for preview + */ + @EngineThread + @NonNull + protected abstract List getPreviewStreamAvailableSizes(); + + @EngineThread + @NonNull + @SuppressWarnings("WeakerAccess") + protected final Size computePreviewStreamSize() { + @NonNull List previewSizes = getPreviewStreamAvailableSizes(); + // These sizes come in REF_SENSOR. Since there is an external selector involved, + // we must convert all of them to REF_VIEW, then flip back when returning. + boolean flip = getAngles().flip(Reference.SENSOR, Reference.VIEW); + List sizes = new ArrayList<>(previewSizes.size()); + for (Size size : previewSizes) { + sizes.add(flip ? size.flip() : size); + } + + // Create our own default selector, which will be used if the external + // mPreviewStreamSizeSelector is null, or if it fails in finding a size. + Size targetMinSize = getPreviewSurfaceSize(Reference.VIEW); + if (targetMinSize == null) { + throw new IllegalStateException("targetMinSize should not be null here."); + } + AspectRatio targetRatio = AspectRatio.of(mCaptureSize.getWidth(), mCaptureSize.getHeight()); + if (flip) targetRatio = targetRatio.flip(); + LOG.i("computePreviewStreamSize:", + "targetRatio:", targetRatio, + "targetMinSize:", targetMinSize); + SizeSelector matchRatio = SizeSelectors.and( // Match this aspect ratio and sort by biggest + SizeSelectors.aspectRatio(targetRatio, 0), + SizeSelectors.biggest()); + SizeSelector matchSize = SizeSelectors.and( // Bigger than this size, and sort by smallest + SizeSelectors.minHeight(targetMinSize.getHeight()), + SizeSelectors.minWidth(targetMinSize.getWidth()), + SizeSelectors.smallest()); + SizeSelector matchAll = SizeSelectors.or( + SizeSelectors.and(matchRatio, matchSize), // Try to respect both constraints. + matchSize, // If couldn't match aspect ratio, at least respect the size + matchRatio, // If couldn't respect size, at least match aspect ratio + SizeSelectors.biggest() // If couldn't match any, take the biggest. + ); + + // Apply the external selector with this as a fallback, + // and return a size in REF_SENSOR reference. + SizeSelector selector; + if (mPreviewStreamSizeSelector != null) { + selector = SizeSelectors.or(mPreviewStreamSizeSelector, matchAll); + } else { + selector = matchAll; + } + Size result = selector.select(sizes).get(0); + if (!sizes.contains(result)) { + throw new RuntimeException("SizeSelectors must not return Sizes other than " + + "those in the input list."); + } + if (flip) result = result.flip(); + LOG.i("computePreviewStreamSize:", "result:", result, "flip:", flip); + return result; + } + + /** + * This is called anytime {@link #computeFrameProcessingSize()} is called. + * Implementors can return null if frame processor size is not selectable + * @return a list of available sizes for frame processing + */ + @EngineThread + @NonNull + protected abstract List getFrameProcessingAvailableSizes(); + + @EngineThread + @NonNull + @SuppressWarnings("WeakerAccess") + protected final Size computeFrameProcessingSize() { + @NonNull List frameSizes = getFrameProcessingAvailableSizes(); + // These sizes come in REF_SENSOR. Since there is an external selector involved, + // we must convert all of them to REF_VIEW, then flip back when returning. + boolean flip = getAngles().flip(Reference.SENSOR, Reference.VIEW); + List sizes = new ArrayList<>(frameSizes.size()); + for (Size size : frameSizes) { + sizes.add(flip ? size.flip() : size); + } + AspectRatio targetRatio = AspectRatio.of( + mPreviewStreamSize.getWidth(), + mPreviewStreamSize.getHeight()); + if (flip) targetRatio = targetRatio.flip(); + int maxWidth = mFrameProcessingMaxWidth; + int maxHeight = mFrameProcessingMaxHeight; + if (maxWidth <= 0 || maxWidth == Integer.MAX_VALUE) maxWidth = 640; + if (maxHeight <= 0 || maxHeight == Integer.MAX_VALUE) maxHeight = 640; + Size targetMaxSize = new Size(maxWidth, maxHeight); + LOG.i("computeFrameProcessingSize:", + "targetRatio:", targetRatio, + "targetMaxSize:", targetMaxSize); + SizeSelector matchRatio = SizeSelectors.aspectRatio(targetRatio, 0); + SizeSelector matchSize = SizeSelectors.and( + SizeSelectors.maxHeight(targetMaxSize.getHeight()), + SizeSelectors.maxWidth(targetMaxSize.getWidth()), + SizeSelectors.biggest()); + SizeSelector matchAll = SizeSelectors.or( + SizeSelectors.and(matchRatio, matchSize), // Try to respect both constraints. + matchSize, // If couldn't match aspect ratio, at least respect the size + SizeSelectors.smallest() // If couldn't match any, take the smallest. + ); + Size result = matchAll.select(sizes).get(0); + if (!sizes.contains(result)) { + throw new RuntimeException("SizeSelectors must not return Sizes other than " + + "those in the input list."); + } + if (flip) result = result.flip(); + LOG.i("computeFrameProcessingSize:", "result:", result, "flip:", flip); + return result; + } + + //endregion +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java index 5f75d43f..837c7331 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java @@ -8,7 +8,6 @@ import android.location.Location; import android.os.Handler; import android.os.Looper; -import com.google.android.gms.tasks.Continuation; import com.google.android.gms.tasks.OnCompleteListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.android.gms.tasks.SuccessContinuation; @@ -39,21 +38,15 @@ import com.otaliastudios.cameraview.controls.Hdr; import com.otaliastudios.cameraview.controls.Mode; import com.otaliastudios.cameraview.controls.VideoCodec; import com.otaliastudios.cameraview.controls.WhiteBalance; -import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.Size; import com.otaliastudios.cameraview.size.SizeSelector; -import com.otaliastudios.cameraview.size.SizeSelectors; import com.otaliastudios.cameraview.video.VideoRecorder; -import androidx.annotation.CallSuper; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -136,53 +129,13 @@ public abstract class CameraEngine implements protected static final String TAG = CameraEngine.class.getSimpleName(); protected static final CameraLogger LOG = CameraLogger.create(TAG); - // If this is 2, this means we'll try to run destroy() twice. private static final int DESTROY_RETRIES = 2; - // Need to be protected - @SuppressWarnings("WeakerAccess") protected final Callback mCallback; - @SuppressWarnings("WeakerAccess") protected CameraPreview mPreview; - @SuppressWarnings("WeakerAccess") protected CameraOptions mCameraOptions; - @SuppressWarnings("WeakerAccess") protected PictureRecorder mPictureRecorder; - @SuppressWarnings("WeakerAccess") protected VideoRecorder mVideoRecorder; - @SuppressWarnings("WeakerAccess") protected Size mCaptureSize; - @SuppressWarnings("WeakerAccess") protected Size mPreviewStreamSize; - @SuppressWarnings("WeakerAccess") protected Flash mFlash; - @SuppressWarnings("WeakerAccess") protected WhiteBalance mWhiteBalance; - @SuppressWarnings("WeakerAccess") protected VideoCodec mVideoCodec; - @SuppressWarnings("WeakerAccess") protected Hdr mHdr; - @SuppressWarnings("WeakerAccess") protected PictureFormat mPictureFormat; - @SuppressWarnings("WeakerAccess") protected Location mLocation; - @SuppressWarnings("WeakerAccess") protected float mZoomValue; - @SuppressWarnings("WeakerAccess") protected float mExposureCorrectionValue; - @SuppressWarnings("WeakerAccess") protected boolean mPlaySounds; - @SuppressWarnings("WeakerAccess") protected boolean mPictureMetering; - @SuppressWarnings("WeakerAccess") protected boolean mPictureSnapshotMetering; - @SuppressWarnings("WeakerAccess") protected float mPreviewFrameRate; - private WorkerHandler mHandler; @VisibleForTesting Handler mCrashHandler; - private final FrameManager mFrameManager; - private final Angles mAngles; - @Nullable private SizeSelector mPreviewStreamSizeSelector; - private SizeSelector mPictureSizeSelector; - private SizeSelector mVideoSizeSelector; - private Facing mFacing; - private Mode mMode; - private Audio mAudio; - private long mVideoMaxSize; - private int mVideoMaxDuration; - private int mVideoBitRate; - private int mAudioBitRate; - private boolean mHasFrameProcessors; - private long mAutoFocusResetDelayMillis; - private int mSnapshotMaxWidth = Integer.MAX_VALUE; // in REF_VIEW like SizeSelectors - private int mSnapshotMaxHeight = Integer.MAX_VALUE; // in REF_VIEW like SizeSelectors - private Overlay overlay; - - @SuppressWarnings("WeakerAccess") - protected final CameraStateOrchestrator mOrchestrator + private final Callback mCallback; + private final CameraStateOrchestrator mOrchestrator = new CameraStateOrchestrator(new CameraOrchestrator.Callback() { @Override @NonNull @@ -192,45 +145,24 @@ public abstract class CameraEngine implements @Override public void handleJobException(@NonNull String job, @NonNull Exception exception) { - handleException(Thread.currentThread(), exception, false); + handleException(exception, false); } }); - // Ops used for testing. - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mZoomTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mExposureCorrectionTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mFlashTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mWhiteBalanceTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mHdrTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mLocationTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mPlaySoundsTask - = Tasks.forResult(null); - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) Task mPreviewFrameRateTask - = Tasks.forResult(null); - protected CameraEngine(@NonNull Callback callback) { mCallback = callback; mCrashHandler = new Handler(Looper.getMainLooper()); - mFrameManager = instantiateFrameManager(); - mAngles = new Angles(); recreateHandler(false); } - public void setPreview(@NonNull CameraPreview cameraPreview) { - if (mPreview != null) mPreview.setSurfaceCallback(null); - mPreview = cameraPreview; - mPreview.setSurfaceCallback(this); + @NonNull + protected final Callback getCallback() { + return mCallback; } @NonNull - public CameraPreview getPreview() { - return mPreview; + protected final CameraStateOrchestrator getOrchestrator() { + return mOrchestrator; } //region Error handling @@ -242,7 +174,7 @@ public abstract class CameraEngine implements private class CrashExceptionHandler implements Thread.UncaughtExceptionHandler { @Override public void uncaughtException(@NonNull Thread thread, @NonNull Throwable throwable) { - handleException(thread, throwable, true); + handleException(throwable, true); } } @@ -267,17 +199,15 @@ public abstract class CameraEngine implements * In the first case, the thread is about to be terminated. In the second case, * we can actually keep using it. * - * @param thread the thread * @param throwable the throwable - * @param fromExceptionHandler true if coming from exception handler + * @param isUncaught true if coming from exception handler */ - private void handleException(@NonNull final Thread thread, - final @NonNull Throwable throwable, - final boolean fromExceptionHandler) { + private void handleException(@NonNull final Throwable throwable, + final boolean isUncaught) { // 1. If this comes from the exception handler, the thread has crashed. Replace it. // Most actions are wrapped into Tasks so don't go here, but some callbacks do // (at least in Camera1, e.g. onError). - if (fromExceptionHandler) { + if (isUncaught) { LOG.e("EXCEPTION:", "Handler thread is gone. Replacing."); recreateHandler(false); } @@ -338,7 +268,7 @@ public abstract class CameraEngine implements return mOrchestrator.getTargetState(); } - public boolean isChangingState() { + public final boolean isChangingState() { return mOrchestrator.hasPendingStateChange(); } @@ -421,7 +351,7 @@ public abstract class CameraEngine implements return stopEngine(swallowExceptions); } - @SuppressWarnings("WeakerAccess") + @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) @NonNull protected Task restartBind() { LOG.i("RESTART BIND:", "scheduled. State:", getState()); @@ -431,7 +361,7 @@ public abstract class CameraEngine implements return startPreview(); } - @SuppressWarnings("WeakerAccess") + @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) @NonNull protected Task restartPreview() { LOG.i("RESTART PREVIEW:", "scheduled. State:", getState()); @@ -451,8 +381,8 @@ public abstract class CameraEngine implements new Callable>() { @Override public Task call() { - if (!collectCameraInfo(mFacing)) { - LOG.e("onStartEngine:", "No camera available for facing", mFacing); + if (!collectCameraInfo(getFacing())) { + LOG.e("onStartEngine:", "No camera available for facing", getFacing()); throw new CameraException(CameraException.REASON_NO_CAMERA); } return onStartEngine(); @@ -490,6 +420,19 @@ public abstract class CameraEngine implements }); } + /** + * Camera is about to be opened. Implementors should look into available cameras + * and see if anyone matches the given {@link Facing value}. + * + * If so, implementors should set {@link Angles#setSensorOffset(Facing, int)} + * and any other information (like camera ID) needed to start the engine. + * + * @param facing the facing value + * @return true if we have one + */ + @EngineThread + protected abstract boolean collectCameraInfo(@NonNull Facing facing); + /** * Starts the engine. * @return a task @@ -520,7 +463,7 @@ public abstract class CameraEngine implements new Callable>() { @Override public Task call() { - if (mPreview != null && mPreview.hasSurface()) { + if (getPreview() != null && getPreview().hasSurface()) { return onStartBind(); } else { return Tasks.forCanceled(); @@ -618,9 +561,10 @@ public abstract class CameraEngine implements * The surface is now available, which means that step 1 has completed. * If we have also completed step 2, go on with binding and streaming. */ + @SuppressWarnings("ConstantConditions") @Override public final void onSurfaceAvailable() { - LOG.i("onSurfaceAvailable:", "Size is", getPreviewSurfaceSize(Reference.VIEW)); + LOG.i("onSurfaceAvailable:", "Size is", getPreview().getSurfaceSize()); startBind(); startPreview(); } @@ -632,720 +576,138 @@ public abstract class CameraEngine implements stopBind(false); } - @Override - public final void onSurfaceChanged() { - LOG.i("onSurfaceChanged:", "Size is", getPreviewSurfaceSize(Reference.VIEW)); - mOrchestrator.scheduleStateful("surface changed", CameraState.BIND, - new Runnable() { - @Override - public void run() { - // Compute a new camera preview size and apply. - Size newSize = computePreviewStreamSize(); - if (newSize.equals(mPreviewStreamSize)) { - LOG.i("onSurfaceChanged:", - "The computed preview size is identical. No op."); - } else { - LOG.i("onSurfaceChanged:", - "Computed a new preview size. Calling onPreviewStreamSizeChanged()."); - mPreviewStreamSize = newSize; - onPreviewStreamSizeChanged(); - } - } - }); - } - - /** - * The preview stream size has changed. At this point, some engine might want to - * simply call {@link #restartPreview()}, others to {@link #restartBind()}. - * - * It basically depends on the step at which the preview stream size is actually used. - */ - @EngineThread - protected abstract void onPreviewStreamSizeChanged(); - //endregion - //region Final setters and getters - - public final void setOverlay(@Nullable Overlay overlay) { - this.overlay = overlay; - } - - @Nullable - public final Overlay getOverlay() { - return overlay; - } - - public final Angles getAngles() { - return mAngles; - } - - public final void setPreviewStreamSizeSelector(@Nullable SizeSelector selector) { - mPreviewStreamSizeSelector = selector; - } - - @Nullable - public final SizeSelector getPreviewStreamSizeSelector() { - return mPreviewStreamSizeSelector; - } - - public final void setPictureSizeSelector(@NonNull SizeSelector selector) { - mPictureSizeSelector = selector; - } + //region Abstract getters @NonNull - public final SizeSelector getPictureSizeSelector() { - return mPictureSizeSelector; - } - - public final void setVideoSizeSelector(@NonNull SizeSelector selector) { - mVideoSizeSelector = selector; - } + public abstract Angles getAngles(); @NonNull - public final SizeSelector getVideoSizeSelector() { - return mVideoSizeSelector; - } - - public final void setVideoMaxSize(long videoMaxSizeBytes) { - mVideoMaxSize = videoMaxSizeBytes; - } - - public final long getVideoMaxSize() { - return mVideoMaxSize; - } - - public final void setVideoMaxDuration(int videoMaxDurationMillis) { - mVideoMaxDuration = videoMaxDurationMillis; - } - - public final int getVideoMaxDuration() { - return mVideoMaxDuration; - } - - public final void setVideoCodec(@NonNull VideoCodec codec) { - mVideoCodec = codec; - } - - public final VideoCodec getVideoCodec() { - return mVideoCodec; - } - - public final void setVideoBitRate(int videoBitRate) { - mVideoBitRate = videoBitRate; - } - - public final int getVideoBitRate() { - return mVideoBitRate; - } - - public final void setAudioBitRate(int audioBitRate) { - mAudioBitRate = audioBitRate; - } + public abstract FrameManager getFrameManager(); - public final int getAudioBitRate() { - return mAudioBitRate; - } - - public final void setSnapshotMaxWidth(int maxWidth) { - mSnapshotMaxWidth = maxWidth; - } - - public int getSnapshotMaxWidth() { - return mSnapshotMaxWidth; - } - - public final void setSnapshotMaxHeight(int maxHeight) { - mSnapshotMaxHeight = maxHeight; - } - - public int getSnapshotMaxHeight() { - return mSnapshotMaxHeight; - } - - public final void setAutoFocusResetDelay(long delayMillis) { - mAutoFocusResetDelayMillis = delayMillis; - } - - public final long getAutoFocusResetDelay() { - return mAutoFocusResetDelayMillis; - } - - /** - * Sets a new facing value. This will restart the engine session (if there's any) - * so that we can open the new facing camera. - * @param facing facing - */ - public final void setFacing(final @NonNull Facing facing) { - final Facing old = mFacing; - if (facing != old) { - mFacing = facing; - mOrchestrator.scheduleStateful("facing", CameraState.ENGINE, - new Runnable() { - @Override - public void run() { - if (collectCameraInfo(facing)) { - restart(); - } else { - mFacing = old; - } - } - }); - } - } - - @NonNull - public final Facing getFacing() { - return mFacing; - } + @Nullable + public abstract CameraOptions getCameraOptions(); - /** - * Sets a new audio value that will be used for video recordings. - * @param audio desired audio - */ - public final void setAudio(@NonNull Audio audio) { - if (mAudio != audio) { - if (isTakingVideo()) { - LOG.w("Audio setting was changed while recording. " + - "Changes will take place starting from next video"); - } - mAudio = audio; - } - } + @Nullable + public abstract Size getPictureSize(@NonNull Reference reference); - @NonNull - public final Audio getAudio() { - return mAudio; - } + @Nullable + public abstract Size getVideoSize(@NonNull Reference reference); - /** - * Sets the desired mode (either picture or video). - * @param mode desired mode. - */ - public final void setMode(@NonNull Mode mode) { - if (mode != mMode) { - mMode = mode; - mOrchestrator.scheduleStateful("mode", CameraState.ENGINE, - new Runnable() { - @Override - public void run() { - restart(); - } - }); - } - } + @Nullable + public abstract Size getPreviewStreamSize(@NonNull Reference reference); - @NonNull - public final Mode getMode() { - return mMode; - } + @Nullable + public abstract Size getUncroppedSnapshotSize(@NonNull Reference reference); - @NonNull - public final FrameManager getFrameManager() { - return mFrameManager; - } + //endregion - @Nullable - public final CameraOptions getCameraOptions() { - return mCameraOptions; - } + //region Abstract APIs - @NonNull - public final Flash getFlash() { - return mFlash; - } + public abstract void setPreview(@NonNull CameraPreview cameraPreview); + @Nullable public abstract CameraPreview getPreview(); - @NonNull - public final WhiteBalance getWhiteBalance() { - return mWhiteBalance; - } + public abstract void setOverlay(@Nullable Overlay overlay); + @Nullable public abstract Overlay getOverlay(); - @NonNull - public final Hdr getHdr() { - return mHdr; - } + public abstract void setPreviewStreamSizeSelector(@Nullable SizeSelector selector); + @Nullable public abstract SizeSelector getPreviewStreamSizeSelector(); - @Nullable - public final Location getLocation() { - return mLocation; - } + public abstract void setPictureSizeSelector(@NonNull SizeSelector selector); + @NonNull public abstract SizeSelector getPictureSizeSelector(); - @NonNull - public final PictureFormat getPictureFormat() { - return mPictureFormat; - } + public abstract void setVideoSizeSelector(@NonNull SizeSelector selector); + @NonNull public abstract SizeSelector getVideoSizeSelector(); - public final float getZoomValue() { - return mZoomValue; - } + public abstract void setVideoMaxSize(long videoMaxSizeBytes); + public abstract long getVideoMaxSize(); - public final float getExposureCorrectionValue() { - return mExposureCorrectionValue; - } + public abstract void setVideoMaxDuration(int videoMaxDurationMillis); + public abstract int getVideoMaxDuration(); - public final float getPreviewFrameRate() { - return mPreviewFrameRate; - } + public abstract void setVideoCodec(@NonNull VideoCodec codec); + @NonNull public abstract VideoCodec getVideoCodec(); - @CallSuper - public void setHasFrameProcessors(boolean hasFrameProcessors) { - mHasFrameProcessors = hasFrameProcessors; - } + public abstract void setVideoBitRate(int videoBitRate); + public abstract int getVideoBitRate(); - @SuppressWarnings("WeakerAccess") - public final boolean hasFrameProcessors() { - return mHasFrameProcessors; - } + public abstract void setAudioBitRate(int audioBitRate); + public abstract int getAudioBitRate(); - @SuppressWarnings("WeakerAccess") - protected final boolean shouldResetAutoFocus() { - return mAutoFocusResetDelayMillis > 0 && mAutoFocusResetDelayMillis != Long.MAX_VALUE; - } + public abstract void setSnapshotMaxWidth(int maxWidth); + public abstract int getSnapshotMaxWidth(); - public final void setPictureMetering(boolean enable) { - mPictureMetering = enable; - } + public abstract void setSnapshotMaxHeight(int maxHeight); + public abstract int getSnapshotMaxHeight(); - public final boolean getPictureMetering() { - return mPictureMetering; - } + public abstract void setFrameProcessingMaxWidth(int maxWidth); + public abstract int getFrameProcessingMaxWidth(); - public final void setPictureSnapshotMetering(boolean enable) { - mPictureSnapshotMetering = enable; - } + public abstract void setFrameProcessingMaxHeight(int maxHeight); + public abstract int getFrameProcessingMaxHeight(); - public final boolean getPictureSnapshotMetering() { - return mPictureSnapshotMetering; - } + public abstract void setFrameProcessingFormat(int format); + public abstract int getFrameProcessingFormat(); - //endregion + public abstract void setAutoFocusResetDelay(long delayMillis); + public abstract long getAutoFocusResetDelay(); - //region Abstract setters and APIs + public abstract void setFacing(final @NonNull Facing facing); + @NonNull public abstract Facing getFacing(); - /** - * Camera is about to be opened. Implementors should look into available cameras - * and see if anyone matches the given {@link Facing value}. - * - * If so, implementors should set {@link Angles#setSensorOffset(Facing, int)} - * and any other information (like camera ID) needed to start the engine. - * - * @param facing the facing value - * @return true if we have one - */ - @EngineThread - protected abstract boolean collectCameraInfo(@NonNull Facing facing); + public abstract void setAudio(@NonNull Audio audio); + @NonNull public abstract Audio getAudio(); - /** - * Called at construction time to get a frame manager that can later be - * accessed through {@link #getFrameManager()}. - * @return a frame manager - */ - @NonNull - protected abstract FrameManager instantiateFrameManager(); + public abstract void setMode(@NonNull Mode mode); + @NonNull public abstract Mode getMode(); - // If closed, no-op. If opened, check supported and apply. public abstract void setZoom(float zoom, @Nullable PointF[] points, boolean notify); + public abstract float getZoomValue(); - // If closed, no-op. If opened, check supported and apply. - public abstract void setExposureCorrection(float EVvalue, - @NonNull float[] bounds, - @Nullable PointF[] points, - boolean notify); + public abstract void setExposureCorrection(float EVvalue, @NonNull float[] bounds, + @Nullable PointF[] points, boolean notify); + public abstract float getExposureCorrectionValue(); - // If closed, keep. If opened, check supported and apply. public abstract void setFlash(@NonNull Flash flash); + @NonNull public abstract Flash getFlash(); - // If closed, keep. If opened, check supported and apply. public abstract void setWhiteBalance(@NonNull WhiteBalance whiteBalance); + @NonNull public abstract WhiteBalance getWhiteBalance(); - // If closed, keep. If opened, check supported and apply. public abstract void setHdr(@NonNull Hdr hdr); + @NonNull public abstract Hdr getHdr(); - // If closed, keep. If opened, check supported and apply. public abstract void setLocation(@Nullable Location location); + @Nullable public abstract Location getLocation(); - // If closed, keep. If opened, check supported and apply. public abstract void setPictureFormat(@NonNull PictureFormat pictureFormat); - - public abstract void startAutoFocus(@Nullable Gesture gesture, @NonNull PointF point); - - public abstract void setPlaySounds(boolean playSounds); + @NonNull public abstract PictureFormat getPictureFormat(); public abstract void setPreviewFrameRate(float previewFrameRate); + public abstract float getPreviewFrameRate(); - //endregion + public abstract void setHasFrameProcessors(boolean hasFrameProcessors); + public abstract boolean hasFrameProcessors(); - //region picture and video control - - public final boolean isTakingPicture() { - return mPictureRecorder != null; - } + public abstract void setPictureMetering(boolean enable); + public abstract boolean getPictureMetering(); - /* not final for tests */ - public void takePicture(final @NonNull PictureResult.Stub stub) { - // Save boolean before scheduling! See how Camera2Engine calls this with a temp value. - final boolean metering = mPictureMetering; - mOrchestrator.scheduleStateful("take picture", CameraState.BIND, - new Runnable() { - @Override - public void run() { - LOG.i("takePicture:", "running. isTakingPicture:", isTakingPicture()); - if (isTakingPicture()) return; - if (mMode == Mode.VIDEO) { - throw new IllegalStateException("Can't take hq pictures while in VIDEO mode"); - } - stub.isSnapshot = false; - stub.location = mLocation; - stub.facing = mFacing; - stub.format = mPictureFormat; - onTakePicture(stub, metering); - } - }); - } + public abstract void setPictureSnapshotMetering(boolean enable); + public abstract boolean getPictureSnapshotMetering(); - /** - * The snapshot size is the {@link #getPreviewStreamSize(Reference)}, but cropped based on the - * view/surface aspect ratio. - * @param stub a picture stub - */ - public final void takePictureSnapshot(final @NonNull PictureResult.Stub stub) { - // Save boolean before scheduling! See how Camera2Engine calls this with a temp value. - final boolean metering = mPictureSnapshotMetering; - mOrchestrator.scheduleStateful("take picture snapshot", CameraState.BIND, - new Runnable() { - @Override - public void run() { - LOG.i("takePictureSnapshot:", "running. isTakingPicture:", isTakingPicture()); - if (isTakingPicture()) return; - stub.location = mLocation; - stub.isSnapshot = true; - stub.facing = mFacing; - stub.format = PictureFormat.JPEG; - // Leave the other parameters to subclasses. - //noinspection ConstantConditions - AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); - onTakePictureSnapshot(stub, ratio, metering); - } - }); - } - - @Override - public void onPictureShutter(boolean didPlaySound) { - mCallback.onShutter(!didPlaySound); - } - - @Override - public void onPictureResult(@Nullable PictureResult.Stub result, @Nullable Exception error) { - mPictureRecorder = null; - if (result != null) { - mCallback.dispatchOnPictureTaken(result); - } else { - LOG.e("onPictureResult", "result is null: something went wrong.", error); - mCallback.dispatchError(new CameraException(error, - CameraException.REASON_PICTURE_FAILED)); - } - } - - public final boolean isTakingVideo() { - return mVideoRecorder != null && mVideoRecorder.isRecording(); - } - - public final void takeVideo(final @NonNull VideoResult.Stub stub, final @NonNull File file) { - mOrchestrator.scheduleStateful("take video", CameraState.BIND, new Runnable() { - @Override - public void run() { - LOG.i("takeVideo:", "running. isTakingVideo:", isTakingVideo()); - if (isTakingVideo()) return; - if (mMode == Mode.PICTURE) { - throw new IllegalStateException("Can't record video while in PICTURE mode"); - } - stub.file = file; - stub.isSnapshot = false; - stub.videoCodec = mVideoCodec; - stub.location = mLocation; - stub.facing = mFacing; - stub.audio = mAudio; - stub.maxSize = mVideoMaxSize; - stub.maxDuration = mVideoMaxDuration; - stub.videoBitRate = mVideoBitRate; - stub.audioBitRate = mAudioBitRate; - onTakeVideo(stub); - } - }); - } - - /** - * @param stub a video stub - * @param file the output file - */ - public final void takeVideoSnapshot(@NonNull final VideoResult.Stub stub, - @NonNull final File file) { - mOrchestrator.scheduleStateful("take video snapshot", CameraState.BIND, - new Runnable() { - @Override - public void run() { - LOG.i("takeVideoSnapshot:", "running. isTakingVideo:", isTakingVideo()); - stub.file = file; - stub.isSnapshot = true; - stub.videoCodec = mVideoCodec; - stub.location = mLocation; - stub.facing = mFacing; - stub.videoBitRate = mVideoBitRate; - stub.audioBitRate = mAudioBitRate; - stub.audio = mAudio; - stub.maxSize = mVideoMaxSize; - stub.maxDuration = mVideoMaxDuration; - //noinspection ConstantConditions - AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); - onTakeVideoSnapshot(stub, ratio); - } - }); - } - - public final void stopVideo() { - mOrchestrator.schedule("stop video", true, new Runnable() { - @Override - public void run() { - LOG.i("stopVideo", "running. isTakingVideo?", isTakingVideo()); - onStopVideo(); - } - }); - } - - @EngineThread - @SuppressWarnings("WeakerAccess") - protected void onStopVideo() { - if (mVideoRecorder != null) { - mVideoRecorder.stop(false); - // Do not null this, so we respond correctly to isTakingVideo(), - // which checks for recorder presence and recorder.isRecording(). - // It will be nulled in onVideoResult. - } - } - - @CallSuper - @Override - public void onVideoResult(@Nullable VideoResult.Stub result, @Nullable Exception exception) { - mVideoRecorder = null; - if (result != null) { - mCallback.dispatchOnVideoTaken(result); - } else { - LOG.e("onVideoResult", "result is null: something went wrong.", exception); - mCallback.dispatchError(new CameraException(exception, - CameraException.REASON_VIDEO_FAILED)); - } - } - - @Override - public void onVideoRecordingStart() { - mCallback.dispatchOnVideoRecordingStart(); - } - - @Override - public void onVideoRecordingEnd() { - mCallback.dispatchOnVideoRecordingEnd(); - } - - @EngineThread - protected abstract void onTakePicture(@NonNull PictureResult.Stub stub, boolean doMetering); - - @EngineThread - protected abstract void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, - @NonNull AspectRatio outputRatio, - boolean doMetering); - - @EngineThread - protected abstract void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, - @NonNull AspectRatio outputRatio); - - @EngineThread - protected abstract void onTakeVideo(@NonNull VideoResult.Stub stub); - - //endregion - - //region Size utilities - - @Nullable - public final Size getPictureSize(@SuppressWarnings("SameParameterValue") @NonNull Reference reference) { - Size size = mCaptureSize; - if (size == null || mMode == Mode.VIDEO) return null; - return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; - } - - @Nullable - public final Size getVideoSize(@SuppressWarnings("SameParameterValue") @NonNull Reference reference) { - Size size = mCaptureSize; - if (size == null || mMode == Mode.PICTURE) return null; - return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; - } - - @Nullable - public final Size getPreviewStreamSize(@NonNull Reference reference) { - Size size = mPreviewStreamSize; - if (size == null) return null; - return getAngles().flip(Reference.SENSOR, reference) ? size.flip() : size; - } - - @SuppressWarnings("SameParameterValue") - @Nullable - private Size getPreviewSurfaceSize(@NonNull Reference reference) { - CameraPreview preview = mPreview; - if (preview == null) return null; - return getAngles().flip(Reference.VIEW, reference) ? preview.getSurfaceSize().flip() - : preview.getSurfaceSize(); - } - - /** - * Returns the snapshot size, but not cropped with the view dimensions, which - * is what we will do before creating the snapshot. However, cropping is done at various - * levels so we don't want to perform the op here. - * - * The base snapshot size is based on PreviewStreamSize (later cropped with view ratio). Why? - * One might be tempted to say that it's the SurfaceSize (which already matches the view ratio). - * - * The camera sensor will capture preview frames with PreviewStreamSize and that's it. Then they - * are hardware-scaled by the preview surface, but this does not affect the snapshot, as the - * snapshot recorder simply creates another surface. - * - * Done tests to ensure that this is true, by using - * 1. small SurfaceSize and biggest() PreviewStreamSize: output is not low quality - * 2. big SurfaceSize and smallest() PreviewStreamSize: output is low quality - * In both cases the result.size here was set to the biggest of the two. - * - * I could not find the same evidence for videos, but I would say that the same things should - * apply, despite the capturing mechanism being different. - * - * @param reference the reference system - * @return the uncropped snapshot size - */ - @Nullable - public final Size getUncroppedSnapshotSize(@NonNull Reference reference) { - Size baseSize = getPreviewStreamSize(reference); - if (baseSize == null) return null; - boolean flip = getAngles().flip(reference, Reference.VIEW); - int maxWidth = flip ? mSnapshotMaxHeight : mSnapshotMaxWidth; - int maxHeight = flip ? mSnapshotMaxWidth : mSnapshotMaxHeight; - float baseRatio = AspectRatio.of(baseSize).toFloat(); - float maxValuesRatio = AspectRatio.of(maxWidth, maxHeight).toFloat(); - if (maxValuesRatio >= baseRatio) { - // Height is the real constraint. - int outHeight = Math.min(baseSize.getHeight(), maxHeight); - int outWidth = (int) Math.floor((float) outHeight * baseRatio); - return new Size(outWidth, outHeight); - } else { - // Width is the real constraint. - int outWidth = Math.min(baseSize.getWidth(), maxWidth); - int outHeight = (int) Math.floor((float) outWidth / baseRatio); - return new Size(outWidth, outHeight); - } - } - - /** - * This is called either on cameraView.start(), or when the underlying surface changes. - * It is possible that in the first call the preview surface has not already computed its - * dimensions. - * But when it does, the {@link CameraPreview.SurfaceCallback} should be called, - * and this should be refreshed. - * - * @return the capture size - */ - @NonNull - @SuppressWarnings("WeakerAccess") - protected final Size computeCaptureSize() { - return computeCaptureSize(mMode); - } - - @SuppressWarnings("WeakerAccess") - protected final Size computeCaptureSize(@NonNull Mode mode) { - // We want to pass stuff into the REF_VIEW reference, not the sensor one. - // This is already managed by CameraOptions, so we just flip again at the end. - boolean flip = getAngles().flip(Reference.SENSOR, Reference.VIEW); - SizeSelector selector; - Collection sizes; - if (mode == Mode.PICTURE) { - selector = mPictureSizeSelector; - sizes = mCameraOptions.getSupportedPictureSizes(); - } else { - selector = mVideoSizeSelector; - sizes = mCameraOptions.getSupportedVideoSizes(); - } - selector = SizeSelectors.or(selector, SizeSelectors.biggest()); - List list = new ArrayList<>(sizes); - Size result = selector.select(list).get(0); - if (!list.contains(result)) { - throw new RuntimeException("SizeSelectors must not return Sizes other than " + - "those in the input list."); - } - LOG.i("computeCaptureSize:", "result:", result, "flip:", flip, "mode:", mode); - if (flip) result = result.flip(); // Go back to REF_SENSOR - return result; - } + public abstract void startAutoFocus(@Nullable Gesture gesture, @NonNull PointF point); - /** - * This is called anytime {@link #computePreviewStreamSize()} is called. - * This means that it should be called during the binding process, when - * we can be sure that the camera is available (engineState == STARTED). - * @return a list of available sizes for preview - */ - @EngineThread - @NonNull - protected abstract List getPreviewStreamAvailableSizes(); + public abstract void setPlaySounds(boolean playSounds); - @EngineThread - @NonNull - @SuppressWarnings("WeakerAccess") - protected final Size computePreviewStreamSize() { - @NonNull List previewSizes = getPreviewStreamAvailableSizes(); - // These sizes come in REF_SENSOR. Since there is an external selector involved, - // we must convert all of them to REF_VIEW, then flip back when returning. - boolean flip = getAngles().flip(Reference.SENSOR, Reference.VIEW); - List sizes = new ArrayList<>(previewSizes.size()); - for (Size size : previewSizes) { - sizes.add(flip ? size.flip() : size); - } + public abstract boolean isTakingPicture(); + public abstract void takePicture(@NonNull PictureResult.Stub stub); + public abstract void takePictureSnapshot(final @NonNull PictureResult.Stub stub); - // Create our own default selector, which will be used if the external - // mPreviewStreamSizeSelector is null, or if it fails in finding a size. - Size targetMinSize = getPreviewSurfaceSize(Reference.VIEW); - if (targetMinSize == null) { - throw new IllegalStateException("targetMinSize should not be null here."); - } - AspectRatio targetRatio = AspectRatio.of(mCaptureSize.getWidth(), mCaptureSize.getHeight()); - if (flip) targetRatio = targetRatio.flip(); - LOG.i("computePreviewStreamSize:", - "targetRatio:", targetRatio, - "targetMinSize:", targetMinSize); - SizeSelector matchRatio = SizeSelectors.and( // Match this aspect ratio and sort by biggest - SizeSelectors.aspectRatio(targetRatio, 0), - SizeSelectors.biggest()); - SizeSelector matchSize = SizeSelectors.and( // Bigger than this size, and sort by smallest - SizeSelectors.minHeight(targetMinSize.getHeight()), - SizeSelectors.minWidth(targetMinSize.getWidth()), - SizeSelectors.smallest()); - SizeSelector matchAll = SizeSelectors.or( - SizeSelectors.and(matchRatio, matchSize), // Try to respect both constraints. - matchSize, // If couldn't match aspect ratio, at least respect the size - matchRatio, // If couldn't respect size, at least match aspect ratio - SizeSelectors.biggest() // If couldn't match any, take the biggest. - ); - - // Apply the external selector with this as a fallback, - // and return a size in REF_SENSOR reference. - SizeSelector selector; - if (mPreviewStreamSizeSelector != null) { - selector = SizeSelectors.or(mPreviewStreamSizeSelector, matchAll); - } else { - selector = matchAll; - } - Size result = selector.select(sizes).get(0); - if (!sizes.contains(result)) { - throw new RuntimeException("SizeSelectors must not return Sizes other than " + - "those in the input list."); - } - if (flip) result = result.flip(); - LOG.i("computePreviewStreamSize:", "result:", result, "flip:", flip); - return result; - } + public abstract boolean isTakingVideo(); + public abstract void takeVideo(@NonNull VideoResult.Stub stub, @NonNull File file); + public abstract void takeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull File file); + public abstract void stopVideo(); //endregion } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera1Options.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera1Options.java index dd8e1a7a..af629578 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera1Options.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera1Options.java @@ -1,5 +1,6 @@ package com.otaliastudios.cameraview.engine.options; +import android.graphics.ImageFormat; import android.hardware.Camera; import android.media.CamcorderProfile; @@ -128,5 +129,8 @@ public class Camera1Options extends CameraOptions { // Picture formats supportedPictureFormats.add(PictureFormat.JPEG); + + // Frame processing formats + supportedFrameProcessingFormats.add(ImageFormat.NV21); } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera2Options.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera2Options.java index 09791608..91a37a0d 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera2Options.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/options/Camera2Options.java @@ -174,5 +174,15 @@ public class Camera2Options extends CameraOptions { } } } + + // Frame processing formats + supportedFrameProcessingFormats.add(ImageFormat.YUV_420_888); + int[] outputFormats = streamMap.getOutputFormats(); + for (int outputFormat : outputFormats) { + // Ensure it is a raw format + if (ImageFormat.getBitsPerPixel(outputFormat) > 0) { + supportedFrameProcessingFormats.add(outputFormat); + } + } } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java new file mode 100644 index 00000000..96ae862c --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java @@ -0,0 +1,162 @@ +package com.otaliastudios.cameraview.frame; + + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.otaliastudios.cameraview.size.Size; + +import java.util.concurrent.LinkedBlockingQueue; + +/** + * This class manages the allocation of byte buffers and {@link Frame} objects. + * We are interested in recycling both of them, especially byte[] buffers which can create a lot + * of overhead. + * + * The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use + * the same number since they are consumed at the same time. + * + * We can work in two modes, depending on whether a + * {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior. + * + * 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to + * the {@link BufferCallback}. The callback should then fill the buffer, and finally call + * {@link FrameManager#getFrame(Object, long, int)} to receive a frame. + * This is used for Camera1. + * + * 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte + * buffers, instead of handing them to the callback. The users can ask for buffers through + * {@link #getBuffer()}. + * This buffer can be filled with data and used to get a frame + * {@link FrameManager#getFrame(Object, long, int)}, or, in case it was not filled, returned to + * the queue using {@link #onBufferUnused(byte[])}. + * This is used for Camera2. + */ +public class ByteBufferFrameManager extends FrameManager { + + /** + * Receives callbacks on buffer availability + * (when a Frame is released, we reuse its buffer). + */ + public interface BufferCallback { + void onBufferAvailable(@NonNull byte[] buffer); + } + + /** + * In this mode, we have a {@link #mBufferCallback} and dispatch + * new buffers to the callback. + */ + private final static int BUFFER_MODE_DISPATCH = 0; + + /** + * In this mode, we have a {@link #mBufferQueue} where we store + * buffers and only dispatch when requested. + */ + private final static int BUFFER_MODE_ENQUEUE = 1; + + private LinkedBlockingQueue mBufferQueue; + private BufferCallback mBufferCallback; + private final int mBufferMode; + + /** + * Construct a new frame manager. + * The construction must be followed by an {@link #setUp(int, Size)} call + * as soon as the parameters are known. + * + * @param poolSize the size of the backing pool. + * @param callback a callback + */ + public ByteBufferFrameManager(int poolSize, @Nullable BufferCallback callback) { + super(poolSize, byte[].class); + if (callback != null) { + mBufferCallback = callback; + mBufferMode = BUFFER_MODE_DISPATCH; + } else { + mBufferQueue = new LinkedBlockingQueue<>(poolSize); + mBufferMode = BUFFER_MODE_ENQUEUE; + } + } + + + @Override + public void setUp(int format, @NonNull Size size) { + super.setUp(format, size); + int bytes = getFrameBytes(); + for (int i = 0; i < getPoolSize(); i++) { + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(new byte[bytes]); + } else { + mBufferQueue.offer(new byte[bytes]); + } + } + } + + /** + * Returns a new byte buffer than can be filled. + * This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame + * manager also holds a queue of the byte buffers. + * + * If not null, the buffer returned by this method can be filled and used to get + * a new frame through {@link FrameManager#getFrame(Object, long, int)}. + * + * @return a buffer, or null + */ + @Nullable + public byte[] getBuffer() { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call getBuffer() " + + "when not in BUFFER_MODE_ENQUEUE."); + } + return mBufferQueue.poll(); + } + + /** + * Can be called if the buffer obtained by {@link #getBuffer()} + * was not used to construct a frame, so it can be put back into the queue. + * @param buffer a buffer + */ + public void onBufferUnused(@NonNull byte[] buffer) { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call onBufferUnused() " + + "when not in BUFFER_MODE_ENQUEUE."); + } + + if (isSetUp()) { + mBufferQueue.offer(buffer); + } else { + LOG.w("onBufferUnused: buffer was returned but we're not set up anymore."); + } + } + + @Override + protected void onFrameDataReleased(@NonNull byte[] data, boolean recycled) { + if (recycled && data.length == getFrameBytes()) { + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(data); + } else { + mBufferQueue.offer(data); + } + } + } + + @NonNull + @Override + protected byte[] onCloneFrameData(@NonNull byte[] data) { + byte[] clone = new byte[data.length]; + System.arraycopy(data, 0, clone, 0, data.length); + return clone; + } + + /** + * Releases all frames controlled by this manager and + * clears the pool. + * In BUFFER_MODE_ENQUEUE, releases also all the buffers. + */ + @Override + public void release() { + super.release(); + if (mBufferMode == BUFFER_MODE_ENQUEUE) { + mBufferQueue.clear(); + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java index 8ee2f97c..1694cb3e 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java @@ -1,6 +1,9 @@ package com.otaliastudios.cameraview.frame; +import android.annotation.SuppressLint; + import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.controls.Engine; import com.otaliastudios.cameraview.size.Size; import androidx.annotation.NonNull; @@ -14,8 +17,9 @@ public class Frame { private final static CameraLogger LOG = CameraLogger.create(TAG); private final FrameManager mManager; + private final Class mDataClass; - private byte[] mData = null; + private Object mData = null; private long mTime = -1; private long mLastTime = -1; private int mRotation = 0; @@ -24,9 +28,10 @@ public class Frame { Frame(@NonNull FrameManager manager) { mManager = manager; + mDataClass = manager.getFrameDataClass(); } - void setContent(@NonNull byte[] data, long time, int rotation, @NonNull Size size, int format) { + void setContent(@NonNull Object data, long time, int rotation, @NonNull Size size, int format) { this.mData = data; this.mTime = time; this.mLastTime = time; @@ -63,12 +68,13 @@ public class Frame { * * @return a frozen Frame */ + @SuppressLint("NewApi") @NonNull public Frame freeze() { ensureHasContent(); - byte[] data = new byte[mData.length]; - System.arraycopy(mData, 0, data, 0, mData.length); Frame other = new Frame(mManager); + //noinspection unchecked + Object data = mManager.cloneFrameData(getData()); other.setContent(data, mTime, mRotation, mSize, mFormat); return other; } @@ -80,7 +86,7 @@ public class Frame { public void release() { if (!hasContent()) return; LOG.v("Frame with time", mTime, "is being released."); - byte[] data = mData; + Object data = mData; mData = null; mRotation = 0; mTime = -1; @@ -89,6 +95,7 @@ public class Frame { // After the manager is notified, this frame instance can be taken by // someone else, possibly from another thread. So this should be the // last call in this method. If we null data after, we can have issues. + //noinspection unchecked mManager.onFrameReleased(this, data); } @@ -96,10 +103,23 @@ public class Frame { * Returns the frame data. * @return the frame data */ + @SuppressWarnings("unchecked") @NonNull - public byte[] getData() { + public T getData() { ensureHasContent(); - return mData; + return (T) mData; + } + + /** + * Returns the class returned by {@link #getData()}. + * This class depends on the engine that produced this frame. + * - {@link Engine#CAMERA1} will produce byte[] arrays + * - {@link Engine#CAMERA2} will produce {@link android.media.Image}s + * @return the data class + */ + @NonNull + public Class getDataClass() { + return mDataClass; } /** diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java index 55f2a5e5..65c095fe 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java @@ -7,76 +7,33 @@ import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.size.Size; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import java.util.concurrent.LinkedBlockingQueue; /** - * This class manages the allocation of byte buffers and {@link Frame} objects. - * We are interested in recycling both of them, especially byte[] buffers which can create a lot - * of overhead. - * + * This class manages the allocation of {@link Frame} objects. * The FrameManager keeps a {@link #mPoolSize} integer that defines the number of instances to keep. - * The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use - * the same number since they are consumed at the same time. * * Main methods are: * - {@link #setUp(int, Size)}: to set up with size and allocate buffers * - {@link #release()}: to release. After release, a manager can be setUp again. - * - {@link #getFrame(byte[], long, int)}: gets a new {@link Frame}. + * - {@link #getFrame(Object, long, int)}: gets a new {@link Frame}. * - * For both byte buffers and frames to get back to the FrameManager pool, all you have to do + * For frames to get back to the FrameManager pool, all you have to do * is call {@link Frame#release()} when done. - * - * Other than this, the FrameManager can work in two modes, depending on whether a - * {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior. - * - * 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to - * the {@link BufferCallback}. The callback should then fill the buffer, and finally call - * {@link #getFrame(byte[], long, int)} to receive a frame. - * This is used for Camera1. - * - * 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte - * buffers, instead of handing them to the callback. The users can ask for buffers through - * {@link #getBuffer()}. - * This buffer can be filled with data and used to get a frame - * {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue - * using {@link #onBufferUnused(byte[])}. - * This is used for Camera2. */ -public class FrameManager { +public abstract class FrameManager { private static final String TAG = FrameManager.class.getSimpleName(); - private static final CameraLogger LOG = CameraLogger.create(TAG); - - /** - * Receives callbacks on buffer availability - * (when a Frame is released, we reuse its buffer). - */ - public interface BufferCallback { - void onBufferAvailable(@NonNull byte[] buffer); - } + protected static final CameraLogger LOG = CameraLogger.create(TAG); private final int mPoolSize; - private int mBufferSize = -1; + private int mFrameBytes = -1; private Size mFrameSize = null; private int mFrameFormat = -1; + private final Class mFrameDataClass; private LinkedBlockingQueue mFrameQueue; - private LinkedBlockingQueue mBufferQueue; - private BufferCallback mBufferCallback; - private final int mBufferMode; - /** - * In this mode, we have a {@link #mBufferCallback} and dispatch - * new buffers to the callback. - */ - private final static int BUFFER_MODE_DISPATCH = 0; - - /** - * In this mode, we have a {@link #mBufferQueue} where we store - * buffers and only dispatch when requested. - */ - private final static int BUFFER_MODE_ENQUEUE = 1; /** * Construct a new frame manager. @@ -84,18 +41,37 @@ public class FrameManager { * as soon as the parameters are known. * * @param poolSize the size of the backing pool. - * @param callback a callback */ - public FrameManager(int poolSize, @Nullable BufferCallback callback) { + protected FrameManager(int poolSize, @NonNull Class dataClass) { mPoolSize = poolSize; + mFrameDataClass = dataClass; mFrameQueue = new LinkedBlockingQueue<>(mPoolSize); - if (callback != null) { - mBufferCallback = callback; - mBufferMode = BUFFER_MODE_DISPATCH; - } else { - mBufferQueue = new LinkedBlockingQueue<>(mPoolSize); - mBufferMode = BUFFER_MODE_ENQUEUE; - } + } + + /** + * Returns the pool size. + * @return pool size + */ + @SuppressWarnings("WeakerAccess") + public final int getPoolSize() { + return mPoolSize; + } + + /** + * Returns the frame size in bytes. + * @return frame size in bytes + */ + @SuppressWarnings("WeakerAccess") + public final int getFrameBytes() { + return mFrameBytes; + } + + /** + * Returns the frame data class. + * @return frame data class + */ + public final Class getFrameDataClass() { + return mFrameDataClass; } /** @@ -106,9 +82,8 @@ public class FrameManager { * * @param format the image format * @param size the frame size - * @return the buffer size */ - public int setUp(int format, @NonNull Size size) { + public void setUp(int format, @NonNull Size size) { if (isSetUp()) { // TODO throw or just reconfigure? } @@ -116,15 +91,7 @@ public class FrameManager { mFrameFormat = format; int bitsPerPixel = ImageFormat.getBitsPerPixel(format); long sizeInBits = size.getHeight() * size.getWidth() * bitsPerPixel; - mBufferSize = (int) Math.ceil(sizeInBits / 8.0d); - for (int i = 0; i < mPoolSize; i++) { - if (mBufferMode == BUFFER_MODE_DISPATCH) { - mBufferCallback.onBufferAvailable(new byte[mBufferSize]); - } else { - mBufferQueue.offer(new byte[mBufferSize]); - } - } - return mBufferSize; + mFrameBytes = (int) Math.ceil(sizeInBits / 8.0d); } /** @@ -134,47 +101,10 @@ public class FrameManager { * * @return true if set up */ - private boolean isSetUp() { + protected boolean isSetUp() { return mFrameSize != null; } - /** - * Returns a new byte buffer than can be filled. - * This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame - * manager also holds a queue of the byte buffers. - * - * If not null, the buffer returned by this method can be filled and used to get - * a new frame through {@link #getFrame(byte[], long, int)}. - * - * @return a buffer, or null - */ - @Nullable - public byte[] getBuffer() { - if (mBufferMode != BUFFER_MODE_ENQUEUE) { - throw new IllegalStateException("Can't call getBuffer() " + - "when not in BUFFER_MODE_ENQUEUE."); - } - return mBufferQueue.poll(); - } - - /** - * Can be called if the buffer obtained by {@link #getBuffer()} - * was not used to construct a frame, so it can be put back into the queue. - * @param buffer a buffer - */ - public void onBufferUnused(@NonNull byte[] buffer) { - if (mBufferMode != BUFFER_MODE_ENQUEUE) { - throw new IllegalStateException("Can't call onBufferUnused() " + - "when not in BUFFER_MODE_ENQUEUE."); - } - - if (isSetUp()) { - mBufferQueue.offer(buffer); - } else { - LOG.w("onBufferUnused: buffer was returned but we're not set up anymore."); - } - } - /** * Returns a new Frame for the given data. This must be called * - after {@link #setUp(int, Size)}, which sets the buffer size @@ -190,7 +120,7 @@ public class FrameManager { * @return a new frame */ @NonNull - public Frame getFrame(@NonNull byte[] data, long time, int rotation) { + public Frame getFrame(@NonNull T data, long time, int rotation) { if (!isSetUp()) { throw new IllegalStateException("Can't call getFrame() after releasing " + "or before setUp."); @@ -209,33 +139,37 @@ public class FrameManager { /** * Called by child frames when they are released. - * This might be called from old Frames that belong to an old 'setUp' - * of this FrameManager instance. So the buffer size might be different, - * for instance. - * * @param frame the released frame */ - void onFrameReleased(@NonNull Frame frame, @NonNull byte[] buffer) { + void onFrameReleased(@NonNull Frame frame, @NonNull T data) { if (!isSetUp()) return; // If frame queue is full, let's drop everything. // If frame queue accepts this frame, let's recycle the buffer as well. - if (mFrameQueue.offer(frame)) { - int currSize = buffer.length; - int reqSize = mBufferSize; - if (currSize == reqSize) { - if (mBufferMode == BUFFER_MODE_DISPATCH) { - mBufferCallback.onBufferAvailable(buffer); - } else { - mBufferQueue.offer(buffer); - } - } - } + boolean recycled = mFrameQueue.offer(frame); + onFrameDataReleased(data, recycled); } + /** + * Called when a Frame was released and its data is now available. + * This might be called from old Frames that belong to an old 'setUp' + * of this FrameManager instance. So the buffer size might be different, + * for instance. + * @param data data + * @param recycled recycled + */ + protected abstract void onFrameDataReleased(@NonNull T data, boolean recycled); + + @NonNull + final T cloneFrameData(@NonNull T data) { + return onCloneFrameData(data); + } + + @NonNull + protected abstract T onCloneFrameData(@NonNull T data); + /** * Releases all frames controlled by this manager and * clears the pool. - * In BUFFER_MODE_ENQUEUE, releases also all the buffers. */ public void release() { if (!isSetUp()) { @@ -245,10 +179,7 @@ public class FrameManager { LOG.i("release: Clearing the frame and buffer queue."); mFrameQueue.clear(); - if (mBufferMode == BUFFER_MODE_ENQUEUE) { - mBufferQueue.clear(); - } - mBufferSize = -1; + mFrameBytes = -1; mFrameSize = null; mFrameFormat = -1; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ImageFrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ImageFrameManager.java new file mode 100644 index 00000000..9a2e774a --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ImageFrameManager.java @@ -0,0 +1,30 @@ +package com.otaliastudios.cameraview.frame; + +import android.media.Image; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +@RequiresApi(Build.VERSION_CODES.KITKAT) +public class ImageFrameManager extends FrameManager { + + public ImageFrameManager(int poolSize) { + super(poolSize, Image.class); + } + + @Override + protected void onFrameDataReleased(@NonNull Image data, boolean recycled) { + try { + data.close(); + } catch (Exception ignore) {} + } + + @NonNull + @Override + protected Image onCloneFrameData(@NonNull Image data) { + throw new RuntimeException("Cannot freeze() an Image Frame. " + + "Please consider using the frame synchronously in your process() method, " + + "which also gives better performance."); + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java deleted file mode 100644 index 6f6127a0..00000000 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.otaliastudios.cameraview.internal.utils; - -import android.graphics.ImageFormat; -import android.media.Image; - -import java.nio.ByteBuffer; - -import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; - -/** - * Conversions for {@link android.media.Image}s into byte arrays. - */ -@RequiresApi(19) -public class ImageHelper { - - /** - * From https://stackoverflow.com/a/52740776/4288782 . - * The result array should have a size that is at least 3/2 * w * h. - * This is correctly computed by {@link com.otaliastudios.cameraview.frame.FrameManager}. - * - * @param image input image - * @param result output array - */ - public static void convertToNV21(@NonNull Image image, @NonNull byte[] result) { - if (image.getFormat() != ImageFormat.YUV_420_888) { - throw new IllegalStateException("CAn only convert from YUV_420_888."); - } - int width = image.getWidth(); - int height = image.getHeight(); - int ySize = width * height; - int uvSize = width * height / 4; - - ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); // Y - ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); // U - ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); // V - - int rowStride = image.getPlanes()[0].getRowStride(); - - if (image.getPlanes()[0].getPixelStride() != 1) { - throw new AssertionError("Something wrong in convertToNV21"); - } - - int pos = 0; - - if (rowStride == width) { // likely - yBuffer.get(result, 0, ySize); - pos += ySize; - } - else { - int yBufferPos = width - rowStride; // not an actual position - for (; pos + + + + diff --git a/cameraview/src/test/java/com/otaliastudios/cameraview/frame/FrameTest.java b/cameraview/src/test/java/com/otaliastudios/cameraview/frame/FrameTest.java index 3f31548b..b8a60dd6 100644 --- a/cameraview/src/test/java/com/otaliastudios/cameraview/frame/FrameTest.java +++ b/cameraview/src/test/java/com/otaliastudios/cameraview/frame/FrameTest.java @@ -3,31 +3,37 @@ package com.otaliastudios.cameraview.frame; import android.graphics.ImageFormat; +import androidx.annotation.NonNull; + import com.otaliastudios.cameraview.size.Size; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static junit.framework.Assert.assertNotNull; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; public class FrameTest { - private FrameManager manager; + private FrameManager manager; @Before public void setUp() { - manager = mock(FrameManager.class); + manager = spy(new FrameManager(1, String.class) { + @Override + protected void onFrameDataReleased(@NonNull String data, boolean recycled) { } + + @NonNull + @Override + protected String onCloneFrameData(@NonNull String data) { + return data; + } + }); } @After @@ -40,21 +46,21 @@ public class FrameTest { // Only time should count. Frame f1 = new Frame(manager); long time = 1000; - f1.setContent(new byte[3], time, 90, new Size(5, 5), ImageFormat.NV21); + f1.setContent("foo", time, 90, new Size(5, 5), ImageFormat.NV21); Frame f2 = new Frame(manager); - f2.setContent(new byte[2], time, 0, new Size(10, 10), ImageFormat.NV21); + f2.setContent("bar", time, 0, new Size(10, 10), ImageFormat.NV21); assertEquals(f1, f2); - f2.setContent(new byte[2], time + 1, 0, new Size(10, 10), ImageFormat.NV21); + f2.setContent("foo", time + 1, 0, new Size(10, 10), ImageFormat.NV21); assertNotEquals(f1, f2); } @Test public void testReleaseThrows() { final Frame frame = new Frame(manager); - frame.setContent(new byte[2], 1000, 90, new Size(10, 10), ImageFormat.NV21); + frame.setContent("foo", 1000, 90, new Size(10, 10), ImageFormat.NV21); frame.release(); - verify(manager, times(1)).onFrameReleased(eq(frame), any(byte[].class)); + verify(manager, times(1)).onFrameReleased(frame, "foo"); assertThrows(new Runnable() { public void run() { frame.getTime(); }}); assertThrows(new Runnable() { public void run() { frame.getFormat(); }}); @@ -75,7 +81,7 @@ public class FrameTest { @Test public void testFreeze() { Frame frame = new Frame(manager); - byte[] data = new byte[]{0, 1, 5, 0, 7, 3, 4, 5}; + String data = "test data"; long time = 1000; int rotation = 90; Size size = new Size(10, 10); @@ -83,14 +89,14 @@ public class FrameTest { frame.setContent(data, time, rotation, size, format); Frame frozen = frame.freeze(); - assertArrayEquals(data, frozen.getData()); + assertEquals(data, frozen.getData()); assertEquals(time, frozen.getTime()); assertEquals(rotation, frozen.getRotation()); assertEquals(size, frozen.getSize()); // Mutate the first, ensure that frozen is not affected - frame.setContent(new byte[]{3, 2, 1}, 50, 180, new Size(1, 1), ImageFormat.JPEG); - assertArrayEquals(data, frozen.getData()); + frame.setContent("new data", 50, 180, new Size(1, 1), ImageFormat.JPEG); + assertEquals(data, frozen.getData()); assertEquals(time, frozen.getTime()); assertEquals(rotation, frozen.getRotation()); assertEquals(size, frozen.getSize()); diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java index 97923046..1ee1891e 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java @@ -47,7 +47,7 @@ import java.util.List; public class CameraActivity extends AppCompatActivity implements View.OnClickListener, OptionView.Callback { private final static CameraLogger LOG = CameraLogger.create("DemoApp"); - private final static boolean USE_FRAME_PROCESSOR = false; + private final static boolean USE_FRAME_PROCESSOR = true; private final static boolean DECODE_BITMAP = true; private CameraView camera; @@ -78,18 +78,24 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis lastTime = newTime; LOG.e("Frame delayMillis:", delay, "FPS:", 1000 / delay); if (DECODE_BITMAP) { - YuvImage yuvImage = new YuvImage(frame.getData(), ImageFormat.NV21, - frame.getSize().getWidth(), - frame.getSize().getHeight(), - null); - ByteArrayOutputStream jpegStream = new ByteArrayOutputStream(); - yuvImage.compressToJpeg(new Rect(0, 0, - frame.getSize().getWidth(), - frame.getSize().getHeight()), 100, jpegStream); - byte[] jpegByteArray = jpegStream.toByteArray(); - Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.length); - //noinspection ResultOfMethodCallIgnored - bitmap.toString(); + if (frame.getFormat() == ImageFormat.NV21 + && frame.getDataClass() == byte[].class) { + byte[] data = frame.getData(); + YuvImage yuvImage = new YuvImage(data, + frame.getFormat(), + frame.getSize().getWidth(), + frame.getSize().getHeight(), + null); + ByteArrayOutputStream jpegStream = new ByteArrayOutputStream(); + yuvImage.compressToJpeg(new Rect(0, 0, + frame.getSize().getWidth(), + frame.getSize().getHeight()), 100, jpegStream); + byte[] jpegByteArray = jpegStream.toByteArray(); + Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, + 0, jpegByteArray.length); + //noinspection ResultOfMethodCallIgnored + bitmap.toString(); + } } } }); @@ -125,6 +131,8 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis new Option.OverlayInPreview(watermark), new Option.OverlayInPictureSnapshot(watermark), new Option.OverlayInVideoSnapshot(watermark), + // Frame Processing + new Option.FrameProcessingFormat(), // Other new Option.Grid(), new Option.GridColor(), new Option.UseDeviceOrientation() ); @@ -141,6 +149,8 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis false, false, false, false, true, // Watermarks false, false, true, + // Frame Processing + true, // Other false, false, true ); diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/Option.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/Option.java index 0b43ca5c..e9297687 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/Option.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/Option.java @@ -4,6 +4,7 @@ import android.graphics.Color; import androidx.annotation.NonNull; import androidx.core.util.Pair; +import android.graphics.ImageFormat; import android.view.View; import android.view.ViewGroup; @@ -554,4 +555,45 @@ public abstract class Option { } } + public static class FrameProcessingFormat extends Option { + + FrameProcessingFormat() { + super("Frame Processing Format"); + } + + @Override + public void set(@NonNull CameraView view, @NonNull Integer value) { + view.setFrameProcessingFormat(value); + } + + @NonNull + @Override + public Integer get(@NonNull CameraView view) { + return view.getFrameProcessingFormat(); + } + + @NonNull + @Override + public Collection getAll(@NonNull CameraView view, @NonNull CameraOptions options) { + return options.getSupportedFrameProcessingFormats(); + } + + @NonNull + @Override + public String toString(@NonNull Integer value) { + switch (value) { + case ImageFormat.NV21: return "NV21"; + case ImageFormat.NV16: return "NV16"; + case ImageFormat.JPEG: return "JPEG"; + case ImageFormat.YUY2: return "YUY2"; + case ImageFormat.YUV_420_888: return "YUV_420_888"; + case ImageFormat.YUV_422_888: return "YUV_422_888"; + case ImageFormat.YUV_444_888: return "YUV_444_888"; + case ImageFormat.RAW10: return "RAW10"; + case ImageFormat.RAW12: return "RAW12"; + case ImageFormat.RAW_SENSOR: return "RAW_SENSOR"; + } + return super.toString(value); + } + } } diff --git a/docs/_posts/2018-12-20-frame-processing.md b/docs/_posts/2018-12-20-frame-processing.md index 87d5714d..6b221d26 100644 --- a/docs/_posts/2018-12-20-frame-processing.md +++ b/docs/_posts/2018-12-20-frame-processing.md @@ -19,12 +19,17 @@ cameraView.addFrameProcessor(new FrameProcessor() { @Override @WorkerThread public void process(@NonNull Frame frame) { - byte[] data = frame.getData(); int rotation = frame.getRotation(); long time = frame.getTime(); Size size = frame.getSize(); int format = frame.getFormat(); - // Process... + if (frame.getDataClass() == byte[].class) { + byte[] data = frame.getData(); + // Process byte array... + } else if (frame.getDataClass() == Image.class) { + Image data = frame.getData(); + // Process android.media.Image... + } } } ``` @@ -78,13 +83,64 @@ public void process(@NonNull Frame frame) { latch.await(); } ``` + +### Frame Data + +Starting from `v2.5.0`, the type of data offered by `frame.getData()` depends on the camera engine +that created this frame: +- The Camera1 engine will offer `byte[]` arrays +- The Camera2 engine will offer `android.media.Image` objects + +You can check this at runtime by inspecting the data class using `frame.getDataClass()`. + +### Frame Size + +The Camera2 engine offers the option to set size constraints for the incoming frames. + +```java +cameraView.setFrameProcessingMaxWidth(maxWidth); +cameraView.setFrameProcessingMaxHeight(maxWidth); +``` + +With other engines, these API have no effect. + +### Frame Format +The Camera2 engine offers the option to set the frame format as one of the ImageFormat +constants. The default is `ImageFormat.YUV_420_888`. + +```java +cameraView.setFrameProcessingFormat(ImageFormat.YUV_420_888); +cameraView.setFrameProcessingFormat(ImageFormat.YUV_422_888); +``` + +With the Camera1 engine, the incoming format will always be `ImageFormat.NV21`. +You can check which formats are available for use through `CameraOptions.getSupportedFrameProcessingFormats()`. + +### XML Attributes + +```xml + +``` + ### Related APIs |Frame API|Type|Description| |---------|----|-----------| |`camera.addFrameProcessor(FrameProcessor)`|`-`|Register a `FrameProcessor`.| -|`frame.getData()`|`byte[]`|The current preview frame, in its original orientation.| +|`camera.removeFrameProcessor(FrameProcessor)`|`-`|Removes a `FrameProcessor`.| +|`camera.clearFrameProcessors()`|`-`|Removes all `FrameProcessor`s.| +|`camera.setFrameProcessingMaxWidth(int)`|`-`|Sets the max width for incoming frames.| +|`camera.setFrameProcessingMaxHeight(int)`|`-`|Sets the max height for incoming frames.| +|`camera.getFrameProcessingMaxWidth()`|`int`|Gets the max width for incoming frames.| +|`camera.getFrameProcessingMaxHeight()`|`int`|Gets the max height for incoming frames.| +|`camera.setFrameProcessingFormat(int)`|`-`|Sets the desired format for incoming frames. Should be one of the ImageFormat constants.| +|`camera.getFrameProcessingFormat()`|`-`|Gets the format for incoming frames. One of the ImageFormat constants.| +|`frame.getDataClass()`|`Class`|The class of the data returned by `getData()`. Either `byte[]` or `android.media.Image`.| +|`frame.getData()`|`T`|The current preview frame, in its original orientation.| |`frame.getTime()`|`long`|The preview timestamp, in `System.currentTimeMillis()` reference.| |`frame.getRotation()`|`int`|The rotation that should be applied to the byte array in order to see what the user sees.| |`frame.getSize()`|`Size`|The frame size, before any rotation is applied, to access data.| diff --git a/docs/_posts/2019-08-06-filters.md b/docs/_posts/2019-08-06-filters.md index 46a07274..d87159f4 100644 --- a/docs/_posts/2019-08-06-filters.md +++ b/docs/_posts/2019-08-06-filters.md @@ -13,10 +13,8 @@ Starting from version `2.1.0`, CameraView experimentally supports real-time filt the camera frames before they are shown and recorded. Just like [overlays](watermarks-and-overlays.html), these filters are applied to the preview and to any [picture or video snapshots](capturing-media.html). -Conditions: - -- you must set the experimental flag: `app:cameraExperimental="true"` -- you must use `Preview.GL_SURFACE` as a preview +Starting from `2.5.0`, this feature is considered to be stable and you do not need the experimental +flag to use it. The only condition is to use the `Preview.GL_SURFACE` preview. ### Simple usage