Frame Processing maxWidth, maxHeight and format (#704)

* Create CameraEngine and CameraBaseEngine * Promote filters to stable - no experimental flag * Fix setSnapshotMaxWidth / Height bugs * Add setFrameProcessingMaxWidth and setFrameProcessingMaxHeight * Add setFrameProcessingMaxWidth and setFrameProcessingMaxHeight (docs) * Prepare Frame for Images, abstract FrameManager, create ByteBufferFrameManager * Fix tests * Fix unit tests * Send Images for Camera2 * Tests * Add CameraView.setFrameProcessingFormat(int), tests, docs * Add CameraOptions.getSupportedFrameProcessingFormats(), tests * Add CameraEngine support, integration tests * Fix demo app, add getFrameProcessingPoolSize * Fix tests * Fix tests
natario1 · Dec 16, 2019 · e1721bb · e1721bb
1 parent 4a6b9be
commit e1721bb
Show file tree

Hide file tree

Showing 30 changed files with 2,054 additions and 1,404 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -64,7 +64,7 @@ jobs:
         with:
           java-version: 1.8
       - name: Execute emulator tests
-        timeout-minutes: 20
+        timeout-minutes: 30
         uses: reactivecircus/[email protected]
         with:
           api-level: ${{ matrix.EMULATOR_API }}

diff --git a/README.md b/README.md
@@ -114,6 +114,9 @@ Using CameraView is extremely simple:
     app:cameraVideoSizeAspectRatio="@string/video_ratio"
     app:cameraSnapshotMaxWidth="@integer/snapshot_max_width"
     app:cameraSnapshotMaxHeight="@integer/snapshot_max_height"
+    app:cameraFrameProcessingMaxWidth="@integer/processing_max_width"
+    app:cameraFrameProcessingMaxHeight="@integer/processing_max_height"
+    app:cameraFrameProcessingFormat="@integer/processing_format"
     app:cameraVideoBitRate="@integer/video_bit_rate"
     app:cameraAudioBitRate="@integer/audio_bit_rate"
     app:cameraGestureTap="none|autoFocus|takePicture"

diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java
@@ -3,6 +3,7 @@
 
 import android.content.Context;
 import android.content.res.TypedArray;
+import android.graphics.ImageFormat;
 import android.graphics.PointF;
 import android.location.Location;
 import androidx.annotation.NonNull;
@@ -168,8 +169,13 @@ public void testDefaults() {
         assertEquals(cameraView.getLocation(), null);
         assertEquals(cameraView.getExposureCorrection(), 0f, 0f);
         assertEquals(cameraView.getZoom(), 0f, 0f);
-        assertEquals(cameraView.getVideoMaxDuration(), 0, 0);
-        assertEquals(cameraView.getVideoMaxSize(), 0, 0);
+        assertEquals(cameraView.getVideoMaxDuration(), 0);
+        assertEquals(cameraView.getVideoMaxSize(), 0);
+        assertEquals(cameraView.getSnapshotMaxWidth(), 0);
+        assertEquals(cameraView.getSnapshotMaxHeight(), 0);
+        assertEquals(cameraView.getFrameProcessingMaxWidth(), 0);
+        assertEquals(cameraView.getFrameProcessingMaxHeight(), 0);
+        assertEquals(cameraView.getFrameProcessingFormat(), 0);
 
         // Self managed
         GestureParser gestures = new GestureParser(empty);
@@ -801,6 +807,30 @@ public void testPreviewFrameRate() {
         assertEquals(cameraView.getPreviewFrameRate(), 60, 0);
     }
 
+    @Test
+    public void testSnapshotMaxSize() {
+        cameraView.setSnapshotMaxWidth(500);
+        assertEquals(500, cameraView.getSnapshotMaxWidth());
+        cameraView.setSnapshotMaxHeight(700);
+        assertEquals(700, cameraView.getSnapshotMaxHeight());
+    }
+
+    @Test
+    public void testFrameProcessingMaxSize() {
+        cameraView.setFrameProcessingMaxWidth(500);
+        assertEquals(500, cameraView.getFrameProcessingMaxWidth());
+        cameraView.setFrameProcessingMaxHeight(700);
+        assertEquals(700, cameraView.getFrameProcessingMaxHeight());
+    }
+
+    @Test
+    public void testFrameProcessingFormat() {
+        cameraView.setFrameProcessingFormat(ImageFormat.YUV_420_888);
+        assertEquals(ImageFormat.YUV_420_888, cameraView.getFrameProcessingFormat());
+        cameraView.setFrameProcessingFormat(ImageFormat.YUV_422_888);
+        assertEquals(ImageFormat.YUV_422_888, cameraView.getFrameProcessingFormat());
+    }
+
     //endregion
 
     //region Lists of listeners and processors
@@ -975,31 +1005,18 @@ public void testOverlays_dontRemoveOverlayView() {
     }
 
     //endregion
-    // TODO: test permissions
 
     //region Filter
 
-    @Test(expected = RuntimeException.class)
-    public void testSetFilter_notExperimental() {
-        cameraView.setExperimental(false);
-        cameraView.setFilter(Filters.AUTO_FIX.newInstance());
-    }
-
-    @Test
-    public void testSetFilter_notExperimental_noFilter() {
-        cameraView.setExperimental(false);
-        cameraView.setFilter(Filters.NONE.newInstance());
-        // no exception thrown
-    }
-
     @Test
     public void testSetFilter() {
-        cameraView.setExperimental(true);
         Filter filter = Filters.AUTO_FIX.newInstance();
         cameraView.setFilter(filter);
         verify(mockPreview, times(1)).setFilter(filter);
         assertEquals(filter, cameraView.getFilter());
         //noinspection ResultOfMethodCallIgnored
         verify(mockPreview, times(1)).getCurrentFilter();
     }
+
+    //endregion
 }
diff --git a/...view/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java b/...view/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera1IntegrationTest.java
@@ -1,6 +1,13 @@
 package com.otaliastudios.cameraview.engine;
 
+import com.otaliastudios.cameraview.CameraLogger;
+import com.otaliastudios.cameraview.CameraOptions;
 import com.otaliastudios.cameraview.controls.Engine;
+import com.otaliastudios.cameraview.frame.Frame;
+import com.otaliastudios.cameraview.frame.FrameProcessor;
+import com.otaliastudios.cameraview.tools.Op;
+import com.otaliastudios.cameraview.tools.Retry;
+import com.otaliastudios.cameraview.tools.SdkExclude;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -10,6 +17,10 @@
 import androidx.test.filters.LargeTest;
 import androidx.test.filters.RequiresDevice;
 
+import java.util.Collection;
+
+import static org.junit.Assert.assertNotNull;
+
 /**
  * These tests work great on real devices, and are the only way to test actual CameraEngine
  * implementation - we really need to open the camera device.
@@ -31,4 +42,10 @@ protected Engine getEngine() {
     protected long getMeteringTimeoutMillis() {
         return Camera1Engine.AUTOFOCUS_END_DELAY_MILLIS;
     }
+
+    @Override
+    public void testFrameProcessing_maxSize() {
+        // Camera1Engine does not support different sizes.
+        // super.testFrameProcessing_maxSize();
+    }
 }
diff --git a/...view/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java b/...view/src/androidTest/java/com/otaliastudios/cameraview/engine/Camera2IntegrationTest.java
@@ -9,6 +9,7 @@
 import com.otaliastudios.cameraview.engine.action.ActionHolder;
 import com.otaliastudios.cameraview.engine.action.BaseAction;
 
+import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import androidx.annotation.NonNull;
@@ -76,4 +77,10 @@ protected boolean canSetVideoMaxDuration() {
         if (shouldOpen) closeSync(true);
         return result;
     }
+
+    @Override
+    public void testFrameProcessing_freezeRelease() {
+        // Camera2 Frames are not freezable.
+        // super.testFrameProcessing_freezeRelease();
+    }
 }
diff --git a/...aview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java b/...aview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java
@@ -51,6 +51,7 @@
 import org.mockito.ArgumentMatcher;
 
 import java.io.File;
+import java.util.Collection;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -70,7 +71,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public abstract class CameraIntegrationTest<E extends CameraEngine> extends BaseTest {
+public abstract class CameraIntegrationTest<E extends CameraBaseEngine> extends BaseTest {
 
     private final static CameraLogger LOG = CameraLogger.create(CameraIntegrationTest.class.getSimpleName());
     private final static long DELAY = 8000;
@@ -1043,6 +1044,27 @@ public void testFrameProcessing_simple() throws Exception {
         assert15Frames(processor);
     }
 
+    @Test
+    @Retry(emulatorOnly = true)
+    @SdkExclude(maxSdkVersion = 22, emulatorOnly = true)
+    public void testFrameProcessing_maxSize() {
+        final int max = 600;
+        camera.setFrameProcessingMaxWidth(max);
+        camera.setFrameProcessingMaxHeight(max);
+        final Op<Size> sizeOp = new Op<>();
+        camera.addFrameProcessor(new FrameProcessor() {
+            @Override
+            public void process(@NonNull Frame frame) {
+                sizeOp.controller().end(frame.getSize());
+            }
+        });
+        openSync(true);
+        Size size = sizeOp.await(2000);
+        assertNotNull(size);
+        assertTrue(size.getWidth() <= max);
+        assertTrue(size.getHeight() <= max);
+    }
+
     @Test
     @Retry(emulatorOnly = true)
     @SdkExclude(maxSdkVersion = 22, emulatorOnly = true)
@@ -1109,6 +1131,35 @@ public void process(@NonNull Frame frame) {
         }
     }
 
+    @Test
+    @Retry(emulatorOnly = true)
+    @SdkExclude(maxSdkVersion = 22, emulatorOnly = true)
+    public void testFrameProcessing_format() {
+        CameraOptions o = openSync(true);
+        Collection<Integer> formats = o.getSupportedFrameProcessingFormats();
+        for (int format : formats) {
+            LOG.i("[TEST FRAME FORMAT]", "Testing", format, "...");
+            Op<Boolean> op = testFrameProcessorFormat(format);
+            assertNotNull(op.await(DELAY));
+        }
+    }
+
+    @NonNull
+    private Op<Boolean> testFrameProcessorFormat(final int format) {
+        final Op<Boolean> op = new Op<>();
+        camera.setFrameProcessingFormat(format);
+        camera.addFrameProcessor(new FrameProcessor() {
+            @Override
+            public void process(@NonNull Frame frame) {
+                if (frame.getFormat() == format) {
+                    op.controller().start();
+                    op.controller().end(true);
+                }
+            }
+        });
+        return op;
+    }
+
     //endregion
 
     //region Overlays

diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java
@@ -13,6 +13,7 @@
 import com.otaliastudios.cameraview.controls.Flash;
 import com.otaliastudios.cameraview.controls.PictureFormat;
 import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
+import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
 import com.otaliastudios.cameraview.frame.FrameManager;
 import com.otaliastudios.cameraview.gesture.Gesture;
 import com.otaliastudios.cameraview.controls.Hdr;
@@ -27,7 +28,7 @@
 import java.util.List;
 import java.util.concurrent.Callable;
 
-public class MockCameraEngine extends CameraEngine {
+public class MockCameraEngine extends CameraBaseEngine {
 
     public boolean mPictureCaptured;
     public boolean mFocusStarted;
@@ -83,7 +84,7 @@ public void setMockPreviewStreamSize(Size size) {
     }
 
     public void setMockState(@NonNull CameraState state) {
-        Task<Void> change = mOrchestrator.scheduleStateChange(getState(),
+        Task<Void> change = getOrchestrator().scheduleStateChange(getState(),
                 state,
                 false,
                 new Callable<Task<Void>>() {
@@ -109,7 +110,6 @@ public void setExposureCorrection(float EVvalue, @NonNull float[] bounds, @Nulla
         mExposureCorrectionChanged = true;
     }
 
-
     @Override
     public void setFlash(@NonNull Flash flash) {
         mFlash = flash;
@@ -135,6 +135,16 @@ public void setPictureFormat(@NonNull PictureFormat pictureFormat) {
         mPictureFormat = pictureFormat;
     }
 
+    @Override
+    public void setHasFrameProcessors(boolean hasFrameProcessors) {
+        mHasFrameProcessors = hasFrameProcessors;
+    }
+
+    @Override
+    public void setFrameProcessingFormat(int format) {
+        mFrameProcessingFormat = format;
+    }
+
     @Override
     public void takePicture(@NonNull PictureResult.Stub stub) {
         super.takePicture(stub);
@@ -172,6 +182,12 @@ protected List<Size> getPreviewStreamAvailableSizes() {
         return new ArrayList<>();
     }
 
+    @NonNull
+    @Override
+    protected List<Size> getFrameProcessingAvailableSizes() {
+        return new ArrayList<>();
+    }
+
     @Override
     public void startAutoFocus(@Nullable Gesture gesture, @NonNull PointF point) {
         mFocusStarted = true;
@@ -180,13 +196,11 @@ public void startAutoFocus(@Nullable Gesture gesture, @NonNull PointF point) {
     @NonNull
     @Override
     protected FrameManager instantiateFrameManager() {
-        return new FrameManager(2, null);
+        return new ByteBufferFrameManager(2, null);
     }
 
     @Override
-    public void setPlaySounds(boolean playSounds) {
-
-    }
+    public void setPlaySounds(boolean playSounds) { }
 
     @Override
     protected boolean collectCameraInfo(@NonNull Facing facing) {

diff --git a/.../src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java b/.../src/androidTest/java/com/otaliastudios/cameraview/engine/options/Camera1OptionsTest.java
@@ -1,13 +1,15 @@
 package com.otaliastudios.cameraview.engine.options;
 
 
+import android.graphics.ImageFormat;
 import android.hardware.Camera;
 
 import com.otaliastudios.cameraview.BaseTest;
 import com.otaliastudios.cameraview.CameraOptions;
 import com.otaliastudios.cameraview.controls.Audio;
 import com.otaliastudios.cameraview.controls.Facing;
 import com.otaliastudios.cameraview.controls.Flash;
+import com.otaliastudios.cameraview.controls.PictureFormat;
 import com.otaliastudios.cameraview.engine.mappers.Camera1Mapper;
 import com.otaliastudios.cameraview.gesture.GestureAction;
 import com.otaliastudios.cameraview.controls.Grid;
@@ -54,6 +56,11 @@ public void testEmpty() {
         assertFalse(o.isZoomSupported());
         assertEquals(o.getExposureCorrectionMaxValue(), 0f, 0);
         assertEquals(o.getExposureCorrectionMinValue(), 0f, 0);
+        // Static
+        assertEquals(1, o.getSupportedPictureFormats().size());
+        assertTrue(o.getSupportedPictureFormats().contains(PictureFormat.JPEG));
+        assertEquals(1, o.getSupportedFrameProcessingFormats().size());
+        assertTrue(o.getSupportedFrameProcessingFormats().contains(ImageFormat.NV21));
     }
 
     private Camera.Size mockCameraSize(int width, int height) {