Create a public ImageProcessor interface
- ImageProcessor is the public API for ImageCapture effect. Hidden for now, and will be made public for API review.
- InternalImageProcessor, an internal wrapper of ImageProcessor/CameraEffect to provide additional info and make sure it's not called on the wrong thread.
Bug: 249593716
Test: manual test and ./gradlew bOS
Change-Id: Ibcccaab4e0e4f4d094c4f29c990f40955ebedc6e
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/CameraEffect.java b/camera/camera-core/src/main/java/androidx/camera/core/CameraEffect.java
index 4d7c8ea..ce8cf2b 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/CameraEffect.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/CameraEffect.java
@@ -15,7 +15,6 @@
*/
package androidx.camera.core;
-import static androidx.camera.core.impl.utils.executor.CameraXExecutors.mainThreadExecutor;
import static androidx.core.util.Preconditions.checkState;
import android.os.Build;
@@ -81,15 +80,22 @@
private final Executor mProcessorExecutor;
@Nullable
private final SurfaceProcessor mSurfaceProcessor;
+ @Nullable
+ private final ImageProcessor mImageProcessor;
/**
- * Private constructor as a workaround to allow @Nullable annotation on final fields.
+ * @param targets the target {@link UseCase} to which this effect should be applied.
+ * @param processorExecutor the {@link Executor} on which the processor will be invoked.
+ * @param imageProcessor a {@link ImageProcessor} implementation.
*/
- @SuppressWarnings("UnusedMethod") // TODO: remove once we add {@link ImageProcessor}.
- private CameraEffect(@Targets int targets) {
+ protected CameraEffect(
+ @Targets int targets,
+ @NonNull Executor processorExecutor,
+ @NonNull ImageProcessor imageProcessor) {
mTargets = targets;
- mProcessorExecutor = mainThreadExecutor();
+ mProcessorExecutor = processorExecutor;
mSurfaceProcessor = null;
+ mImageProcessor = imageProcessor;
}
/**
@@ -104,6 +110,7 @@
mTargets = targets;
mProcessorExecutor = processorExecutor;
mSurfaceProcessor = surfaceProcessor;
+ mImageProcessor = null;
}
/**
@@ -135,6 +142,16 @@
}
/**
+ * Gets the {@link ImageProcessor} associated with this effect.
+ *
+ * <p>This method returns the value set via {@link Builder#setImageProcessor}.
+ */
+ @Nullable
+ public ImageProcessor getImageProcessor() {
+ return mImageProcessor;
+ }
+
+ /**
* Builder class for {@link CameraEffect}.
*/
public static class Builder {
@@ -144,6 +161,8 @@
private Executor mProcessorExecutor;
@Nullable
private SurfaceProcessor mSurfaceProcessor;
+ @Nullable
+ private ImageProcessor mImageProcessor;
/**
* @param targets the target {@link UseCase} of the Effect. e.g. if the
@@ -161,6 +180,9 @@
* {@link SurfaceProcessor} on the {@link Executor}, and deliver the processed output
* frames to the app.
*
+ * <p>Only one processor can be set via {@code #setImageProcessor()} /
+ * {@code #setSurfaceProcessor}, or the {@link #build()} call will throw error.
+ *
* @param executor on which the {@link SurfaceProcessor} will be invoked.
* @param processor the post processor to be injected into CameraX pipeline.
*/
@@ -173,6 +195,27 @@
}
/**
+ * Sets a {@link ImageProcessor} for the effect.
+ *
+ * <p>Once the effect is active, CameraX will send original camera frames to the
+ * {@link ImageProcessor} on the {@link Executor}, and deliver the processed output
+ * frames to the app.
+ *
+ * <p>Only one processor can be set via {@code #setImageProcessor()} /
+ * {@code #setSurfaceProcessor}, or the {@link #build()} call will throw error.
+ *
+ * @param executor on which the {@link ImageProcessor} will be invoked.
+ * @param processor the post processor to be injected into CameraX pipeline.
+ */
+ @NonNull
+ public Builder setImageProcessor(@NonNull Executor executor,
+ @NonNull ImageProcessor processor) {
+ mProcessorExecutor = executor;
+ mImageProcessor = processor;
+ return this;
+ }
+
+ /**
* Builds a {@link CameraEffect} instance.
*
* <p>CameraX supports a selected set of configuration/processor combinations. This method
@@ -181,9 +224,14 @@
*/
@NonNull
public CameraEffect build() {
- checkState(mProcessorExecutor != null && mSurfaceProcessor != null,
- "Must set a processor.");
- return new CameraEffect(mTargets, mProcessorExecutor, mSurfaceProcessor);
+ checkState(mProcessorExecutor != null, "Must have a executor");
+ checkState(mImageProcessor != null ^ mSurfaceProcessor != null,
+ "Must have one and only one processor");
+ if (mSurfaceProcessor != null) {
+ return new CameraEffect(mTargets, mProcessorExecutor, mSurfaceProcessor);
+ } else {
+ return new CameraEffect(mTargets, mProcessorExecutor, mImageProcessor);
+ }
}
}
}
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/ImageProcessor.java b/camera/camera-core/src/main/java/androidx/camera/core/ImageProcessor.java
index 03d7ef7..3ac9d76 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/ImageProcessor.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/ImageProcessor.java
@@ -16,14 +16,129 @@
package androidx.camera.core;
+import android.graphics.PixelFormat;
+
+import androidx.annotation.AnyThread;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
import androidx.annotation.RestrictTo;
+import java.util.List;
+import java.util.concurrent.Executor;
+
/**
- * Interface for injecting a {@link ImageProxy}-based post-processing effect into CameraX.
+ * Interface for injecting a {@link ImageProxy} effect into CameraX.
+ *
+ * <p>Implement {@link ImageProcessor} to inject an effect into CameraX pipeline. For example, to
+ * edit the {@link ImageCapture} result, add a {@link CameraEffect} with the
+ * {@link ImageProcessor} targeting {@link CameraEffect#IMAGE_CAPTURE}. Once injected,
+ * {@link ImageCapture} forwards camera frames to the implementation, and delivers the processed
+ * frames to the app.
+ *
+ * <p>Code sample for creating a {@link ImageCapture} object:
+ * <pre><code>
+ * class ImageEffect implements CameraEffect {
+ * ImageEffect(Executor executor, ImageProcessor imageProcessorImpl) {
+ * super(IMAGE_CAPTURE, executor, imageProcessorImpl);
+ * }
+ * }
+ * </code></pre>
+ *
+ * <p>Code sample for injecting the effect into CameraX pipeline:
+ * <pre><code>
+ * UseCaseGroup useCaseGroup = UseCaseGroup.Builder()
+ * .addUseCase(imageCapture)
+ * .addEffect(new ImageEffect())
+ * .build();
+ * cameraProvider.bindToLifecycle(lifecycleOwner, cameraFilter, useCaseGroup);
+ * </code></pre>
*
* @hide
*/
@RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
public interface ImageProcessor {
- // TODO(b/229629890): create the public interface for post-processing images.
+
+ /**
+ * Accepts original frames from CameraX and returns processed frames.
+ *
+ * <p>CameraX invokes this method for each batch of images from the camera. It's invoked on the
+ * {@link Executor} provided in {@link CameraEffect}'s constructor. It might be called in
+ * parallel, should the {@link Executor} allow multi-threading. The implementation must block
+ * the current calling thread until the output image is returned.
+ *
+ * <p>The implementation must follow the instruction in the {@link Request} to process the
+ * image. For example, it must produce an output image with the format following the JavaDoc of
+ * {@link Request#getInputImages()}. Failing to do so might cause the processing to
+ * fail. For example, for {@link ImageCapture}, when the processing fails, the app will
+ * receive a {@link ImageCapture.OnImageSavedCallback#onError} or
+ * {@link ImageCapture.OnImageCapturedCallback#onError} callback.
+ *
+ * <p>The implementation should throw exceptions if it runs into any unrecoverable errors.
+ * CameraX will catch the error and deliver it to the app via the error callbacks.
+ *
+ * @param request a {@link Request} that contains original images.
+ * @return a {@link Response} that contains processed image.
+ */
+ @NonNull
+ Response process(@NonNull Request request);
+
+ /**
+ * A request for processing one or many {@link ImageProxy}.
+ */
+ interface Request {
+
+ /**
+ * Gets the input images from Camera.
+ *
+ * <p>It may return a single image captured by the camera, or multiple images from a
+ * burst of capture depending on the configuration in {@link CameraEffect}.
+ *
+ * <p>Currently this method only returns a single image.
+ *
+ * @return one or many input images.
+ */
+ @NonNull
+ @AnyThread
+ List<ImageProxy> getInputImages();
+
+ /**
+ * Gets the output image format.
+ *
+ * <p>The {@link Response}'s {@link ImageProxy} must follow the instruction in this
+ * JavaDoc, or CameraX may throw error.
+ *
+ * <p>For {@link PixelFormat#RGBA_8888}, the output image must contain a single plane
+ * with a pixel stride of 4 and a row stride of width * 4. e.g. each pixel is stored on 4
+ * bytes and each RGBA channel is stored with 8 bits of precision. For more details, see the
+ * JavaDoc of {@code Bitmap.Config#ARGB_8888}.
+ *
+ * <p>Currently this method only returns {@link PixelFormat#RGBA_8888}.
+ */
+ @AnyThread
+ int getOutputFormat();
+ }
+
+ /**
+ * A response for injecting an {@link ImageProxy} back to CameraX.
+ */
+ interface Response {
+
+ /**
+ * Gets the output image of the {@link ImageProcessor}
+ *
+ * <p>{@link ImageProcessor} should implement the {@link ImageProxy} and
+ * {@link ImageProxy.PlaneProxy} interfaces to create the {@link ImageProxy} instance.
+ * CameraX will inject the image back to the processing pipeline.
+ *
+ * <p>The {@link ImageProxy} must follow the instruction in the request, or CameraX may
+ * throw error. For example, the format must match the value of
+ * {@link Request#getOutputFormat()}, and the pixel stride must match the description for
+ * that format in {@link Request#getOutputFormat()}'s JavaDoc.
+ *
+ * @return the output image.
+ */
+ @Nullable
+ @AnyThread
+ ImageProxy getOutputImage();
+ }
}
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/processing/ImageProcessorRequest.java b/camera/camera-core/src/main/java/androidx/camera/core/processing/ImageProcessorRequest.java
new file mode 100644
index 0000000..5d5af9c
--- /dev/null
+++ b/camera/camera-core/src/main/java/androidx/camera/core/processing/ImageProcessorRequest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.core.processing;
+
+import android.os.Build;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.RequiresApi;
+import androidx.camera.core.ImageProcessor;
+import androidx.camera.core.ImageProxy;
+
+import java.util.List;
+
+/**
+ * Internal implementation of {@link ImageProcessor.Request} for sending {@link ImageProxy} to
+ * effect implementations.
+ */
+@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+public class ImageProcessorRequest implements ImageProcessor.Request {
+ @NonNull
+ private final List<ImageProxy> mImageProxies;
+ private final int mOutputFormat;
+
+ public ImageProcessorRequest(@NonNull List<ImageProxy> imageProxies, int outputFormat) {
+ mImageProxies = imageProxies;
+ mOutputFormat = outputFormat;
+ }
+
+ @NonNull
+ @Override
+ public List<ImageProxy> getInputImages() {
+ return mImageProxies;
+ }
+
+ @Override
+ public int getOutputFormat() {
+ return mOutputFormat;
+ }
+}
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/processing/InternalImageProcessor.java b/camera/camera-core/src/main/java/androidx/camera/core/processing/InternalImageProcessor.java
new file mode 100644
index 0000000..7db28130
--- /dev/null
+++ b/camera/camera-core/src/main/java/androidx/camera/core/processing/InternalImageProcessor.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.core.processing;
+
+import static androidx.camera.core.ImageCapture.ERROR_UNKNOWN;
+import static androidx.core.util.Preconditions.checkArgument;
+
+import static java.util.Objects.requireNonNull;
+
+import android.os.Build;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.RequiresApi;
+import androidx.camera.core.CameraEffect;
+import androidx.camera.core.ImageCaptureException;
+import androidx.camera.core.ImageProcessor;
+import androidx.concurrent.futures.CallbackToFutureAdapter;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+
+/**
+ * An internal {@link ImageProcessor} that wraps a {@link CameraEffect} targeting
+ * {@link CameraEffect#IMAGE_CAPTURE}.
+ *
+ * <p>This class wrap calls to {@link ImageProcessor} with the effect-provided {@link Executor}.
+ * It also provides additional from Camera
+ */
+@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+public class InternalImageProcessor {
+
+ @NonNull
+ private final Executor mExecutor;
+ @NonNull
+ private final ImageProcessor mImageProcessor;
+
+ public InternalImageProcessor(@NonNull CameraEffect cameraEffect) {
+ checkArgument(cameraEffect.getTargets() == CameraEffect.IMAGE_CAPTURE);
+ mExecutor = cameraEffect.getProcessorExecutor();
+ mImageProcessor = requireNonNull(cameraEffect.getImageProcessor());
+ }
+
+ /**
+ * Forwards the call to {@link ImageProcessor#process} on the effect-provided executor.
+ */
+ @NonNull
+ public ImageProcessor.Response safeProcess(@NonNull ImageProcessor.Request request)
+ throws ImageCaptureException {
+ try {
+ return CallbackToFutureAdapter.getFuture(
+ (CallbackToFutureAdapter.Resolver<ImageProcessor.Response>) completer -> {
+ mExecutor.execute(() -> {
+ try {
+ completer.set(mImageProcessor.process(request));
+ } catch (Exception e) {
+ // Catch all exceptions and forward it CameraX.
+ completer.setException(e);
+ }
+ });
+ return "InternalImageProcessor#process " + request.hashCode();
+ }).get();
+ } catch (ExecutionException | InterruptedException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ throw new ImageCaptureException(
+ ERROR_UNKNOWN, "Failed to invoke ImageProcessor.", cause);
+ }
+ }
+}
diff --git a/camera/camera-core/src/test/java/androidx/camera/core/processing/InternalImageProcessorTest.kt b/camera/camera-core/src/test/java/androidx/camera/core/processing/InternalImageProcessorTest.kt
new file mode 100644
index 0000000..325bb1b
--- /dev/null
+++ b/camera/camera-core/src/test/java/androidx/camera/core/processing/InternalImageProcessorTest.kt
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.core.processing
+
+import android.graphics.PixelFormat
+import androidx.camera.core.CameraEffect
+import androidx.camera.core.ImageCaptureException
+import androidx.camera.core.ImageProcessor
+import androidx.camera.core.ImageProcessor.Response
+import androidx.camera.core.impl.utils.executor.CameraXExecutors.highPriorityExecutor
+import androidx.camera.testing.fakes.FakeImageInfo
+import androidx.camera.testing.fakes.FakeImageProxy
+import com.google.common.truth.Truth.assertThat
+import java.lang.Thread.currentThread
+import java.util.concurrent.Executors.newSingleThreadExecutor
+import org.junit.Assert.fail
+import org.junit.Test
+
+/**
+ * Unit tests for [InternalImageProcessor].
+ */
+class InternalImageProcessorTest {
+
+ companion object {
+ private const val THREAD_NAME = "thread_name"
+ }
+
+ @Test
+ fun processorThrowsError_errorIsPropagatedToCameraX() {
+ // Arrange.
+ val exception = RuntimeException()
+ val cameraEffect = CameraEffect.Builder(CameraEffect.IMAGE_CAPTURE)
+ .setImageProcessor(highPriorityExecutor()) { throw exception }
+ .build()
+ val imageProcessor = InternalImageProcessor(cameraEffect)
+
+ // Act.
+ try {
+ imageProcessor.safeProcess(
+ ImageProcessorRequest(
+ listOf(FakeImageProxy(FakeImageInfo())),
+ PixelFormat.RGBA_8888
+ )
+ )
+ fail("Processor should throw exception")
+ } catch (ex: ImageCaptureException) {
+ // Assert.
+ assertThat(ex.cause).isEqualTo(exception)
+ }
+ }
+
+ @Test
+ fun process_appCallbackInvokedOnAppExecutor() {
+ // Arrange.
+ val imageToEffect = FakeImageProxy(FakeImageInfo())
+ val imageFromEffect = FakeImageProxy(FakeImageInfo())
+ var calledThreadName = ""
+ val processor = ImageProcessor {
+ calledThreadName = currentThread().name
+ Response { imageFromEffect }
+ }
+ val executor = newSingleThreadExecutor { Thread(it, THREAD_NAME) }
+ val cameraEffect = CameraEffect.Builder(CameraEffect.IMAGE_CAPTURE)
+ .setImageProcessor(executor, processor)
+ .build()
+ val imageProcessor = InternalImageProcessor(cameraEffect)
+
+ // Act.
+ val outputImage = imageProcessor.safeProcess(
+ ImageProcessorRequest(listOf(imageToEffect), PixelFormat.RGBA_8888)
+ ).outputImage
+
+ // Assert.
+ assertThat(outputImage).isEqualTo(imageFromEffect)
+ assertThat(calledThreadName).isEqualTo(THREAD_NAME)
+ }
+}
\ No newline at end of file
