|
|
@@ -0,0 +1,242 @@
|
|
|
+/*
|
|
|
+ * Copyright 2020 Google LLC. All rights reserved.
|
|
|
+ *
|
|
|
+ * Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
+ * you may not use this file except in compliance with the License.
|
|
|
+ * You may obtain a copy of the License at
|
|
|
+ *
|
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+ *
|
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+ * See the License for the specific language governing permissions and
|
|
|
+ * limitations under the License.
|
|
|
+ */
|
|
|
+
|
|
|
+package com.example.myapplication;
|
|
|
+
|
|
|
+import android.graphics.SurfaceTexture;
|
|
|
+import android.os.Bundle;
|
|
|
+import android.util.Log;
|
|
|
+import android.util.Size;
|
|
|
+import android.view.SurfaceHolder;
|
|
|
+import android.view.SurfaceView;
|
|
|
+import android.view.View;
|
|
|
+import android.view.ViewGroup;
|
|
|
+import androidx.appcompat.app.AppCompatActivity;
|
|
|
+import com.google.mediapipe.components.CameraHelper;
|
|
|
+import com.google.mediapipe.components.CameraXPreviewHelper;
|
|
|
+import com.google.mediapipe.components.ExternalTextureConverter;
|
|
|
+import com.google.mediapipe.components.FrameProcessor;
|
|
|
+import com.google.mediapipe.components.PermissionHelper;
|
|
|
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
|
|
|
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
|
|
|
+import com.google.mediapipe.framework.AndroidAssetUtil;
|
|
|
+import com.google.mediapipe.framework.PacketGetter;
|
|
|
+import com.google.mediapipe.glutil.EglManager;
|
|
|
+import com.google.protobuf.InvalidProtocolBufferException;
|
|
|
+
|
|
|
+import android.content.pm.ApplicationInfo;
|
|
|
+import android.content.pm.PackageManager;
|
|
|
+import android.content.pm.PackageManager.NameNotFoundException;
|
|
|
+
|
|
|
+
|
|
|
+/**
|
|
|
+ * Main activity of MediaPipe example apps.
|
|
|
+ */
|
|
|
+public class MainActivity extends AppCompatActivity {
|
|
|
+ private static final String TAG = "MainActivity";
|
|
|
+ private static final String BINARY_GRAPH_NAME = "pose_tracking_gpu.binarypb";
|
|
|
+ private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
|
|
|
+ private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
|
|
|
+ private static final String OUTPUT_LANDMARKS_STREAM_NAME = "pose_landmarks";
|
|
|
+ // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
|
|
|
+ // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
|
|
|
+ // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
|
|
|
+ // corner, whereas MediaPipe in general assumes the image origin is at top-left.
|
|
|
+ private static final boolean FLIP_FRAMES_VERTICALLY = true;
|
|
|
+
|
|
|
+ static {
|
|
|
+ // Load all native libraries needed by the app.
|
|
|
+ System.loadLibrary("mediapipe_jni");
|
|
|
+ System.loadLibrary("opencv_java3");
|
|
|
+ }
|
|
|
+
|
|
|
+ // {@link SurfaceTexture} where the camera-preview frames can be accessed.
|
|
|
+ private SurfaceTexture previewFrameTexture;
|
|
|
+ // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
|
|
|
+ private SurfaceView previewDisplayView;
|
|
|
+ // Creates and manages an {@link EGLContext}.
|
|
|
+ private EglManager eglManager;
|
|
|
+ // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
|
|
|
+ // frames onto a {@link Surface}.
|
|
|
+ private FrameProcessor processor;
|
|
|
+ // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
|
|
|
+ // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
|
|
|
+ private ExternalTextureConverter converter;
|
|
|
+ // ApplicationInfo for retrieving metadata defined in the manifest.
|
|
|
+ private ApplicationInfo applicationInfo;
|
|
|
+ // Handles camera access via the {@link CameraX} Jetpack support library.
|
|
|
+ private CameraXPreviewHelper cameraHelper;
|
|
|
+
|
|
|
+ private ResultHandler resultHandler;
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void onCreate(Bundle savedInstanceState) {
|
|
|
+ super.onCreate(savedInstanceState);
|
|
|
+ setContentView(getContentViewLayoutResId());
|
|
|
+
|
|
|
+ resultHandler = new ResultHandler(this);
|
|
|
+
|
|
|
+ previewDisplayView = new SurfaceView(this);
|
|
|
+ setupPreviewDisplayView();
|
|
|
+
|
|
|
+
|
|
|
+ try {
|
|
|
+ applicationInfo =
|
|
|
+ getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
|
|
|
+ } catch (NameNotFoundException e) {
|
|
|
+ Log.e(TAG, "Cannot find application info: " + e);
|
|
|
+ }
|
|
|
+ // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
|
|
|
+ // binary graphs.
|
|
|
+ AndroidAssetUtil.initializeNativeAssetManager(this);
|
|
|
+ eglManager = new EglManager(null);
|
|
|
+ processor =
|
|
|
+ new FrameProcessor(
|
|
|
+ this,
|
|
|
+ eglManager.getNativeContext(),
|
|
|
+ BINARY_GRAPH_NAME,
|
|
|
+ INPUT_VIDEO_STREAM_NAME,
|
|
|
+ OUTPUT_VIDEO_STREAM_NAME);
|
|
|
+ processor
|
|
|
+ .getVideoSurfaceOutput()
|
|
|
+ .setFlipY(FLIP_FRAMES_VERTICALLY);
|
|
|
+
|
|
|
+ processor.addPacketCallback(
|
|
|
+ OUTPUT_LANDMARKS_STREAM_NAME,
|
|
|
+ (packet) -> {
|
|
|
+ try {
|
|
|
+ byte[] landmarksRaw = PacketGetter.getProtoBytes(packet);
|
|
|
+ NormalizedLandmarkList poseLandmarks = NormalizedLandmarkList.parseFrom(landmarksRaw);
|
|
|
+ NormalizedLandmark lm = poseLandmarks.getLandmark(0);
|
|
|
+ resultHandler.handlePosition(lm.getX(), lm.getY());
|
|
|
+ } catch (InvalidProtocolBufferException exception) {
|
|
|
+ Log.e(TAG, "failed to get proto.", exception);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ );
|
|
|
+ PermissionHelper.checkAndRequestCameraPermissions(this);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Used to obtain the content view for this application. If you are extending this class, and
|
|
|
+ // have a custom layout, override this method and return the custom layout.
|
|
|
+ protected int getContentViewLayoutResId() {
|
|
|
+ return R.layout.activity_main;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void onResume() {
|
|
|
+ super.onResume();
|
|
|
+ converter =
|
|
|
+ new ExternalTextureConverter(
|
|
|
+ eglManager.getContext(), 2);
|
|
|
+ converter.setFlipY(FLIP_FRAMES_VERTICALLY);
|
|
|
+ converter.setConsumer(processor);
|
|
|
+ if (PermissionHelper.cameraPermissionsGranted(this)) {
|
|
|
+ startCamera();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void onPause() {
|
|
|
+ super.onPause();
|
|
|
+ converter.close();
|
|
|
+
|
|
|
+ // Hide preview display until we re-open the camera again.
|
|
|
+ previewDisplayView.setVisibility(View.GONE);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onRequestPermissionsResult(
|
|
|
+ int requestCode, String[] permissions, int[] grantResults) {
|
|
|
+ super.onRequestPermissionsResult(requestCode, permissions, grantResults);
|
|
|
+ PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
|
|
|
+ }
|
|
|
+
|
|
|
+ protected void onCameraStarted(SurfaceTexture surfaceTexture) {
|
|
|
+ previewFrameTexture = surfaceTexture;
|
|
|
+
|
|
|
+ // Make the display view visible to start showing the preview. This triggers the
|
|
|
+ // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
|
|
|
+ previewDisplayView.setVisibility(View.VISIBLE);
|
|
|
+ }
|
|
|
+
|
|
|
+ protected Size cameraTargetResolution() {
|
|
|
+ return null; // No preference and let the camera (helper) decide.
|
|
|
+ }
|
|
|
+
|
|
|
+ public void startCamera() {
|
|
|
+ cameraHelper = new CameraXPreviewHelper();
|
|
|
+ cameraHelper.setOnCameraStartedListener(
|
|
|
+ surfaceTexture -> {
|
|
|
+ onCameraStarted(surfaceTexture);
|
|
|
+ });
|
|
|
+ CameraHelper.CameraFacing cameraFacing = CameraHelper.CameraFacing.FRONT;
|
|
|
+ cameraHelper.startCamera(
|
|
|
+ this, cameraFacing,
|
|
|
+ /*unusedSurfaceTexture=*/ null, cameraTargetResolution());
|
|
|
+ }
|
|
|
+
|
|
|
+ protected Size computeViewSize(int width, int height) {
|
|
|
+ return new Size(width, height);
|
|
|
+ }
|
|
|
+
|
|
|
+ protected void onPreviewDisplaySurfaceChanged(
|
|
|
+ SurfaceHolder holder, int format, int width, int height) {
|
|
|
+ // (Re-)Compute the ideal size of the camera-preview display (the area that the
|
|
|
+ // camera-preview frames get rendered onto, potentially with scaling and rotation)
|
|
|
+ // based on the size of the SurfaceView that contains the display.
|
|
|
+ Size viewSize = computeViewSize(width, height);
|
|
|
+ Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
|
|
|
+ boolean isCameraRotated = cameraHelper.isCameraRotated();
|
|
|
+
|
|
|
+ Log.v(TAG, "Meaning the size of the phone display screen ("
|
|
|
+ + displaySize.getWidth() + "," + displaySize.getHeight() + ")");
|
|
|
+
|
|
|
+ // Connect the converter to the camera-preview frames as its input (via
|
|
|
+ // previewFrameTexture), and configure the output width and height as the computed
|
|
|
+ // display size.
|
|
|
+ converter.setSurfaceTextureAndAttachToGLContext(
|
|
|
+ previewFrameTexture,
|
|
|
+ isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
|
|
|
+ isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
|
|
|
+ }
|
|
|
+
|
|
|
+ private void setupPreviewDisplayView() {
|
|
|
+ previewDisplayView.setVisibility(View.GONE);
|
|
|
+ ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
|
|
|
+ viewGroup.addView(previewDisplayView);
|
|
|
+ previewDisplayView
|
|
|
+ .getHolder()
|
|
|
+ .addCallback(
|
|
|
+ new SurfaceHolder.Callback() {
|
|
|
+ @Override
|
|
|
+ public void surfaceCreated(SurfaceHolder holder) {
|
|
|
+ processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
|
|
|
+ Log.d("Surface","Surface Created");
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
|
|
|
+ onPreviewDisplaySurfaceChanged(holder, format, width, height);
|
|
|
+ Log.d("Surface","Surface Changed");
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public void surfaceDestroyed(SurfaceHolder holder) {
|
|
|
+ processor.getVideoSurfaceOutput().setSurface(null);
|
|
|
+ Log.d("Surface","Surface destroy");
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+}
|