Part google-ar#1

- set up the raw depth API

part0_work/src/main/java/com/google/ar/core/codelab/rawdepth/DepthData.java

@@ -0,0 +1,134 @@
+package com.google.ar.core.codelab.rawdepth;
+
+import android.media.Image;
+import android.opengl.Matrix;
+import com.google.ar.core.Anchor;
+import com.google.ar.core.CameraIntrinsics;
+import com.google.ar.core.Frame;
+import com.google.ar.core.exceptions.NotYetAvailableException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+import java.nio.ShortBuffer;
+
+/**
+ * Convert depth data from ARCore depth images to 3D pointclouds. Points are added by calling the
+ * Raw Depth API, and reprojected into 3D space.
+ */
+public class DepthData {
+
+  public static final int FLOATS_PER_POINT = 4; // X,Y,Z,confidence.
+
+  public static FloatBuffer create(Frame frame, Anchor cameraPoseAnchor) {
+    try {
+      Image depthImage = frame.acquireRawDepthImage16Bits();
+      Image confidenceImage = frame.acquireRawDepthConfidenceImage();
+
+      // Retrieve the intrinsic camera parameters corresponding to the depth image to
+      // transform 2D depth pixels into 3D points. See more information about the depth values
+      // at
+      // https://developers.google.com/ar/develop/java/depth/overview#understand-depth-values.
+
+      final CameraIntrinsics intrinsics = frame.getCamera().getTextureIntrinsics();
+      float[] modelMatrix = new float[16];
+      cameraPoseAnchor.getPose().toMatrix(modelMatrix, 0);
+      final FloatBuffer points = convertRawDepthImagesTo3dPointBuffer(
+          depthImage, confidenceImage, intrinsics, modelMatrix);
+
+      depthImage.close();
+      confidenceImage.close();
+
+      return points;
+    } catch (NotYetAvailableException e) {
+      // This normally means that depth data is not available yet.
+      // This is normal, so you don't have to spam the logcat with this.
+    }
+    return null;
+  }
+
+  /** Apply camera intrinsics to convert depth image into a 3D pointcloud. */
+  private static FloatBuffer convertRawDepthImagesTo3dPointBuffer(
+      Image depth, Image confidence, CameraIntrinsics cameraTextureIntrinsics, float[] modelMatrix) {
+    // Java uses big endian so change the endianness to ensure
+    // that the depth data is in the correct byte order.
+    final Image.Plane depthImagePlane = depth.getPlanes()[0];
+    ByteBuffer depthByteBufferOriginal = depthImagePlane.getBuffer();
+    ByteBuffer depthByteBuffer = ByteBuffer.allocate(depthByteBufferOriginal.capacity());
+    depthByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
+    while (depthByteBufferOriginal.hasRemaining()) {
+      depthByteBuffer.put(depthByteBufferOriginal.get());
+    }
+    depthByteBuffer.rewind();
+    ShortBuffer depthBuffer = depthByteBuffer.asShortBuffer();
+
+    final Image.Plane confidenceImagePlane = confidence.getPlanes()[0];
+    ByteBuffer confidenceBufferOriginal = confidenceImagePlane.getBuffer();
+    ByteBuffer confidenceBuffer = ByteBuffer.allocate(confidenceBufferOriginal.capacity());
+    confidenceBuffer.order(ByteOrder.LITTLE_ENDIAN);
+    while (confidenceBufferOriginal.hasRemaining()) {
+      confidenceBuffer.put(confidenceBufferOriginal.get());
+    }
+    confidenceBuffer.rewind();
+
+    // To transform 2D depth pixels into 3D points, retrieve the intrinsic camera parameters
+    // corresponding to the depth image. See more information about the depth values at
+    // https://developers.google.com/ar/develop/java/depth/overview#understand-depth-values.
+    final int[] intrinsicsDimensions = cameraTextureIntrinsics.getImageDimensions();
+    final int depthWidth = depth.getWidth();
+    final int depthHeight = depth.getHeight();
+    final float fx =
+        cameraTextureIntrinsics.getFocalLength()[0] * depthWidth / intrinsicsDimensions[0];
+    final float fy =
+        cameraTextureIntrinsics.getFocalLength()[1] * depthHeight / intrinsicsDimensions[1];
+    final float cx =
+        cameraTextureIntrinsics.getPrincipalPoint()[0] * depthWidth / intrinsicsDimensions[0];
+    final float cy =
+        cameraTextureIntrinsics.getPrincipalPoint()[1] * depthHeight / intrinsicsDimensions[1];
+
+    // Allocate the destination point buffer. If the number of depth pixels is larger than
+    // `maxNumberOfPointsToRender` we uniformly subsample. The raw depth image may have
+    // different resolutions on different devices.
+    final float maxNumberOfPointsToRender = 20000;
+    int step = (int) Math.ceil(Math.sqrt(depthWidth * depthHeight / maxNumberOfPointsToRender));
+
+    FloatBuffer points = FloatBuffer.allocate(depthWidth / step * depthHeight / step * FLOATS_PER_POINT);
+    float[] pointCamera = new float[4];
+    float[] pointWorld = new float[4];
+
+    for (int y = 0; y < depthHeight; y += step) {
+      for (int x = 0; x < depthWidth; x += step) {
+        // Depth images are tightly packed, so it's OK to not use row and pixel strides.
+        int depthMillimeters = depthBuffer.get(y * depthWidth + x); // Depth image pixels are in mm.
+        if (depthMillimeters == 0) {
+          // Pixels with value zero are invalid, meaning depth estimates are missing from
+          // this location.
+          continue;
+        }
+        final float depthMeters = depthMillimeters / 1000.0f; // Depth image pixels are in mm.
+
+        // Retrieve the confidence value for this pixel.
+        final byte confidencePixelValue =
+            confidenceBuffer.get(
+                y * confidenceImagePlane.getRowStride()
+                    + x * confidenceImagePlane.getPixelStride());
+        final float confidenceNormalized = ((float) (confidencePixelValue & 0xff)) / 255.0f;
+
+        // Unproject the depth into a 3D point in camera coordinates.
+        pointCamera[0] = depthMeters * (x - cx) / fx;
+        pointCamera[1] = depthMeters * (cy - y) / fy;
+        pointCamera[2] = -depthMeters;
+        pointCamera[3] = 1;
+
+        // Apply model matrix to transform point into world coordinates.
+        Matrix.multiplyMV(pointWorld, 0, modelMatrix, 0, pointCamera, 0);
+        points.put(pointWorld[0]); // X.
+        points.put(pointWorld[1]); // Y.
+        points.put(pointWorld[2]); // Z.
+        points.put(confidenceNormalized);
+      }
+    }
+
+    points.rewind();
+    return points;
+  }
+}

part0_work/src/main/java/com/google/ar/core/codelab/rawdepth/RawDepthCodelabActivity.java

@@ -114,6 +114,12 @@ protected void onResume() {
 
         // Creates the ARCore session.
         session = new Session(/* context= */ this);
+        if (!session.isDepthModeSupported(Config.DepthMode.RAW_DEPTH_ONLY)) {
+          message =
+              "This device does not support the ARCore Raw Depth API. See" +
+                  "https://developers.google.com/ar/devices for "
+                  + "a list of devices that do.";
+        }
       } catch (UnavailableArcoreNotInstalledException
           | UnavailableUserDeclinedInstallationException e) {
         message = "Please install ARCore";
@@ -140,6 +146,12 @@ protected void onResume() {
     }
 
     try {
+      // Enable raw depth estimation and auto focus mode while ARCore is running.
+      Config config = session.getConfig();
+      config.setDepthMode(Config.DepthMode.RAW_DEPTH_ONLY);
+      config.setFocusMode(Config.FocusMode.AUTO);
+      session.configure(config);
+
       session.resume();
     } catch (CameraNotAvailableException e) {
       messageSnackbarHelper.showError(this, "Camera not available. Try restarting the app.");
@@ -228,6 +240,16 @@ public void onDrawFrame(GL10 gl) {
       // If frame is ready, render camera preview image to the GL surface.
       backgroundRenderer.draw(frame);
 
+      // Retrieve the depth data for this frame.
+      FloatBuffer points = DepthData.create(frame, session.createAnchor(camera.getPose()));
+      if (points == null) {
+        return;
+      }
+
+      if (messageSnackbarHelper.isShowing() && points != null) {
+        messageSnackbarHelper.hide(this);
+      }
+
       // If not tracking, show tracking failure reason instead.
       if (camera.getTrackingState() == TrackingState.PAUSED) {
         messageSnackbarHelper.showMessage(