From 368d71d6a598a61d88fa6f22819746df0da4ec18 Mon Sep 17 00:00:00 2001
From: Sergio Villar Senin <svillar@igalia.com>
Date: Wed, 6 Dec 2023 17:53:46 +0100
Subject: [PATCH] [VisionGlass] Render Wolvic in glasses and control UI on
 phone screen

This is a large change because it's difficult to split it in smaller
chunks as we're revamping the way Wolvic works in the VisionGlass.
In general this is not a good idea but provided that we're in early
stages of VisionGlass port this should be fine as there should not
be regressions as the previous funtionality was quite limited.

These are the main changes:
   * Wolvic is now rendered just in the external screen (the glasses)
     using Android's Presentation API.
   * The phone screen shows a different UI which consists on a quite
     minimalistic and simple UI that allows users to performs clicks
     on a touchpad and also a "back" button.

Apart from that there are some other changes:
   * Fix a crash with early setControllerOrientation call
   * Enable WebXR by properly setting EyeResolution
   * Enable clicks on WebXR experiences
   * Use the actual FOV of the device
   * Improved the "wait for USB permissions" code
---
 .../main/res/layout/visionglass_layout.xml    |  35 +-
 .../visionglass_presentation_layout.xml       |  16 +
 app/src/visionglass/AndroidManifest.xml       |   3 +-
 .../cpp/DeviceDelegateVisionGlass.cpp         | 114 +++---
 .../cpp/DeviceDelegateVisionGlass.h           |   1 +
 app/src/visionglass/cpp/native-lib.cpp        |  13 +-
 .../com/igalia/wolvic/PlatformActivity.java   | 328 +++++++++++++-----
 7 files changed, 335 insertions(+), 175 deletions(-)
 create mode 100644 app/src/main/res/layout/visionglass_presentation_layout.xml

diff --git a/app/src/main/res/layout/visionglass_layout.xml b/app/src/main/res/layout/visionglass_layout.xml
index 7f603b8c405..2bc0ec8c4da 100644
--- a/app/src/main/res/layout/visionglass_layout.xml
+++ b/app/src/main/res/layout/visionglass_layout.xml
@@ -1,26 +1,31 @@
 <?xml version="1.0" encoding="utf-8"?>
-<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:app="http://schemas.android.com/apk/res-auto"
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
     android:layout_width="match_parent"
     android:layout_height="match_parent">
 
-    <android.opengl.GLSurfaceView
-        android:id="@+id/gl_view"
+    <View
+        android:id="@+id/touchpad"
         android:layout_width="match_parent"
         android:layout_height="match_parent"
-        app:layout_constraintBottom_toBottomOf="parent"
-        app:layout_constraintEnd_toEndOf="parent"
-        app:layout_constraintStart_toStartOf="parent"
-        app:layout_constraintTop_toTopOf="parent" />
+        android:background="#333333" />
 
-    <TextView
-        android:id="@+id/frame_rate_text"
+    <Button
+        android:id="@+id/back_button"
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
-        android:layout_marginStart="24dp"
-        android:background="#000000"
-        app:layout_constraintStart_toStartOf="@+id/gl_view"
-        app:layout_constraintTop_toTopOf="@+id/gl_view" />
+        android:text="@string/back_button"
+        android:drawableStart="@drawable/ic_icon_back"
+        android:layout_alignParentBottom="true"
+        android:layout_alignParentStart="true" />
 
+    <Button
+        android:id="@+id/home_button"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="@string/home_tooltip"
+        android:drawableStart="@drawable/ic_icon_home"
+        android:layout_alignParentBottom="true"
+        android:layout_alignParentEnd="true"
+        android:visibility="gone"/>
 
-</androidx.constraintlayout.widget.ConstraintLayout>
+</RelativeLayout>
\ No newline at end of file
diff --git a/app/src/main/res/layout/visionglass_presentation_layout.xml b/app/src/main/res/layout/visionglass_presentation_layout.xml
new file mode 100644
index 00000000000..b37685bef7d
--- /dev/null
+++ b/app/src/main/res/layout/visionglass_presentation_layout.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <android.opengl.GLSurfaceView
+        android:id="@+id/gl_presentation_view"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toTopOf="parent" />
+</androidx.constraintlayout.widget.ConstraintLayout>
diff --git a/app/src/visionglass/AndroidManifest.xml b/app/src/visionglass/AndroidManifest.xml
index 8628fc48721..0c1824463f3 100644
--- a/app/src/visionglass/AndroidManifest.xml
+++ b/app/src/visionglass/AndroidManifest.xml
@@ -7,8 +7,7 @@
 
     <application>
         <activity
-            android:name=".VRBrowserActivity"
-            android:screenOrientation="landscape">
+            android:name=".VRBrowserActivity">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
diff --git a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp
index fdf5d9fbe96..271fbc09f2f 100644
--- a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp
+++ b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp
@@ -34,6 +34,8 @@ jmethodID sSetRenderMode;
 
 namespace crow {
 
+static const float kHorizontalFOV = 36.0f;
+static const float kVerticalFOV = 21.0f;
 static const int32_t kControllerIndex = 0;
 static const vrb::Vector& GetHomePosition() {
   static vrb::Vector homePosition(0.0f, 1.55f, 3.0f);
@@ -43,12 +45,13 @@ static const vrb::Vector& GetHomePosition() {
 struct DeviceDelegateVisionGlass::State {
   vrb::RenderContextWeak context;
   device::RenderMode renderMode;
-  ImmersiveDisplayPtr display;
+  ImmersiveDisplayPtr immersiveDisplay;
   ControllerDelegatePtr controller;
   vrb::CameraEyePtr cameras[2];
   vrb::Color clearColor;
   vrb::Matrix headingMatrix;
   vrb::Vector position;
+  vrb::Quaternion controllerOrientation;
   bool clicked;
   GLsizei glWidth, glHeight;
   float near, far;
@@ -78,21 +81,22 @@ struct DeviceDelegateVisionGlass::State {
   }
 
   void UpdateDisplay() {
-    if (!display)
+    if (!immersiveDisplay)
       return;
 
     vrb::Matrix fov = vrb::Matrix::PerspectiveMatrixWithResolutionDegrees(glWidth, glHeight,
-                                                                          60.0f, -1.0f,
+                                                                          kHorizontalFOV, kVerticalFOV,
                                                                           near, far);
     float left(0.0f), right(0.0f), top(0.0f), bottom(0.0f), n2(0.0f), f2(0.0f);
     fov.DecomposePerspectiveDegrees(left, right, top, bottom, n2, f2);
 
     cameras[0]->SetPerspective(fov);
     cameras[1]->SetPerspective(fov);
-    display->SetFieldOfView(device::Eye::Left, left, right, top, bottom);
-    display->SetFieldOfView(device::Eye::Right, left, right, top, bottom);
+    immersiveDisplay->SetEyeResolution((int32_t)(glWidth / 2), glHeight);
+    immersiveDisplay->SetFieldOfView(device::Eye::Left, left, right, top, bottom);
+    immersiveDisplay->SetFieldOfView(device::Eye::Right, left, right, top, bottom);
 
-    display->SetCapabilityFlags(device::Position | device::Orientation | device::Present | device::InlineSession | device::ImmersiveVRSession);
+    immersiveDisplay->SetCapabilityFlags(device::Orientation | device::Present | device::InlineSession | device::ImmersiveVRSession);
   }
 };
 
@@ -134,11 +138,11 @@ DeviceDelegateVisionGlass::GetRenderMode() {
 
 void
 DeviceDelegateVisionGlass::RegisterImmersiveDisplay(ImmersiveDisplayPtr aDisplay) {
-  m.display = aDisplay;
-  if (m.display) {
-    m.display->SetDeviceName("Vision Glass");
+  m.immersiveDisplay = aDisplay;
+  if (m.immersiveDisplay) {
+    m.immersiveDisplay->SetDeviceName("Vision Glass");
     m.UpdateDisplay();
-    m.display->CompleteEnumeration();
+    m.immersiveDisplay->CompleteEnumeration();
   }
 }
 
@@ -187,13 +191,16 @@ DeviceDelegateVisionGlass::SetClipPlanes(const float aNear, const float aFar) {
 void
 DeviceDelegateVisionGlass::SetControllerDelegate(ControllerDelegatePtr& aController) {
   m.controller = aController;
-  m.controller->CreateController(kControllerIndex, -1, "Oculus Touch (Right)"); // "Wolvic Virtual Controller");
+  m.controller->CreateController(kControllerIndex, kControllerIndex, "Vision Glass Controller");
   m.controller->SetEnabled(kControllerIndex, true);
-  m.controller->SetCapabilityFlags(kControllerIndex, device::Orientation | device::Position);
-  m.controller->SetButtonCount(kControllerIndex, 5);
+  m.controller->SetCapabilityFlags(kControllerIndex, device::Orientation);
   m.controller->SetTargetRayMode(kControllerIndex, device::TargetRayMode::TrackedPointer);
-  static const float data[2] = {0.0f, 0.0f};
-  m.controller->SetAxes(kControllerIndex, data, 2);
+  m.controller->SetControllerType(kControllerIndex, device::VisionGlass);
+  m.controller->SetMode(kControllerIndex, ControllerMode::Device);
+  m.controller->SetAimEnabled(kControllerIndex, true);
+  m.controller->SetCapabilityFlags(kControllerIndex, device::Orientation);
+
+  m.controller->SetButtonCount(kControllerIndex, 5);
 }
 
 void
@@ -223,22 +230,28 @@ DeviceDelegateVisionGlass::StartFrame(const FramePrediction aPrediction) {
   VRB_GL_CHECK(glEnable(GL_BLEND));
   VRB_GL_CHECK(glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT));
   mShouldRender = true;
-  if (m.controller) {
-    vrb::RenderContextPtr context = m.context.lock();
-    if (context) {
-      float level = 100.0 - std::fmod(context->GetTimestamp(), 100.0);
-      m.controller->SetBatteryLevel(kControllerIndex, (int32_t)level);
-    }
-  }
 
-  const float IPD = 0;
-  m.cameras[0]->SetEyeTransform(vrb::Matrix::Translation(vrb::Vector(-IPD * 0.5f, 0.f, 0.f)));
-  m.cameras[1]->SetEyeTransform(vrb::Matrix::Translation(vrb::Vector(IPD * 0.5f, 0.f, 0.f)));
+  const float IPD = 0.064f;
   auto headTransform = m.headingMatrix.Translate(m.position);
   m.cameras[0]->SetHeadTransform(headTransform);
   m.cameras[1]->SetHeadTransform(headTransform);
-  m.display->SetEyeOffset(device::Eye::Left, -IPD * 0.5f, 0.f, 0.f);
-  m.display->SetEyeOffset(device::Eye::Right, IPD * 0.5f, 0.f, 0.f);
+  m.cameras[0]->SetEyeTransform(vrb::Matrix::Translation(vrb::Vector(-IPD * 0.5f, 0.f, 0.f)));
+  m.cameras[1]->SetEyeTransform(vrb::Matrix::Translation(vrb::Vector(IPD * 0.5f, 0.f, 0.f)));
+  m.immersiveDisplay->SetEyeTransform(device::Eye::Left, m.cameras[0]->GetEyeTransform());
+  m.immersiveDisplay->SetEyeTransform(device::Eye::Right, m.cameras[1]->GetEyeTransform());
+
+  // Update controller
+  if (!m.controller)
+    return;
+
+  if (auto context = m.context.lock()) {
+    float level = 100.0 - std::fmod(context->GetTimestamp(), 100.0);
+    m.controller->SetBatteryLevel(kControllerIndex, (int32_t)level);
+  }
+  auto transformMatrix = vrb::Matrix::Rotation(m.controllerOrientation);
+  m.controller->SetTransform(kControllerIndex, transformMatrix);
+  m.controller->SetBeamTransform(kControllerIndex, vrb::Matrix::Identity());
+  m.controller->SetImmersiveBeamTransform(kControllerIndex, vrb::Matrix::Identity());
 }
 
 void
@@ -317,48 +330,6 @@ Clamp(const float aValue) {
   return aValue;
 }
 
-void
-DeviceDelegateVisionGlass::TouchEvent(const bool aDown, const float aX, const float aY) {
-  static const vrb::Vector sForward(0.0f, 0.0f, -1.0f);
-  if (!m.controller) {
-    return;
-  }
-  if (m.renderMode == device::RenderMode::Immersive) {
-    m.controller->SetButtonState(kControllerIndex, ControllerDelegate::BUTTON_TOUCHPAD, 0, false, false);
-    m.clicked = false;
-  } else if (aDown != m.clicked) {
-    m.controller->SetButtonState(kControllerIndex, ControllerDelegate::BUTTON_TOUCHPAD, 0, aDown, aDown);
-    m.clicked = aDown;
-  }
-
-  const float viewportWidth = m.glWidth;
-  const float viewportHeight = m.glHeight;
-  if ((viewportWidth <= 0.0f) || (viewportHeight <= 0.0f)) {
-    return;
-  }
-  const float xModifier = (m.renderMode == device::RenderMode::Immersive ? viewportWidth / 2.0f : 0.0f);
-  const float width = Clamp((((aX - xModifier) / viewportWidth) * 2.0f) - 1.0f);
-  const float height = (((viewportHeight - aY) / viewportHeight) * 2.0f) - 1.0f;
-
-  vrb::Vector start(width, height, -1.0f);
-  vrb::Vector end(width, height, 1.0f);
-  vrb::Matrix inversePerspective = m.cameras[0]->GetPerspective().Inverse();
-  start = inversePerspective.MultiplyPosition(start);
-  end = inversePerspective.MultiplyPosition(end);
-  vrb::Matrix view = m.cameras[0]->GetTransform();
-  start = view.MultiplyPosition(start);
-  end = view.MultiplyPosition(end);
-  const vrb::Vector direction = (end - start).Normalize();
-  const vrb::Vector up = sForward.Cross(direction);
-  const float angle = acosf(sForward.Dot(direction));
-  vrb::Matrix transform = vrb::Matrix::Rotation(up, angle);
-  if (m.renderMode == device::RenderMode::Immersive) {
-    start += direction * 0.3f;
-  }
-  transform.TranslateInPlace(start);
-  m.controller->SetTransform(kControllerIndex, transform);
-}
-
 void
 DeviceDelegateVisionGlass::ControllerButtonPressed(const bool aDown) {
   if (!m.controller) {
@@ -380,6 +351,11 @@ DeviceDelegateVisionGlass::setHead(const float aX, const float aY, const float a
   m.headingMatrix = vrb::Matrix::Rotation({aX,aY,-aZ,-aW});
 }
 
+void
+DeviceDelegateVisionGlass::setControllerOrientation(const float aX, const float aY, const float aZ, const float aW) {
+    m.controllerOrientation = vrb::Quaternion(aX, aY, aZ, aW);
+}
+
 DeviceDelegateVisionGlass::DeviceDelegateVisionGlass(State& aState) : m(aState) {}
 DeviceDelegateVisionGlass::~DeviceDelegateVisionGlass() { m.Shutdown(); }
 
diff --git a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h
index 29000e1bfd3..283b49461db 100644
--- a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h
+++ b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h
@@ -50,6 +50,7 @@ class DeviceDelegateVisionGlass : public DeviceDelegate {
   void TouchEvent(const bool aDown, const float aX, const float aY);
   void ControllerButtonPressed(const bool aDown);
   void setHead(const float aX, const float aY, const float aZ, const float aW);
+  void setControllerOrientation(const float aX, const float aY, const float aZ, const float aW);
 protected:
   struct State;
   DeviceDelegateVisionGlass(State& aState);
diff --git a/app/src/visionglass/cpp/native-lib.cpp b/app/src/visionglass/cpp/native-lib.cpp
index e49ff2e9533..0a3e3122acd 100644
--- a/app/src/visionglass/cpp/native-lib.cpp
+++ b/app/src/visionglass/cpp/native-lib.cpp
@@ -85,14 +85,25 @@ JNI_METHOD(void, drawGL)
 
 JNI_METHOD(void, touchEvent)
 (JNIEnv*, jobject, jboolean aDown, jfloat aX, jfloat aY) {
-    sAppContext->mDevice->TouchEvent(aDown, aX, aY);
+    sAppContext->mDevice->ControllerButtonPressed(aDown);
 }
 
 JNI_METHOD(void, setHead)
 (JNIEnv*, jobject, jdouble aX, jdouble aY, jdouble aZ, jdouble aW) {
+    if (!sAppContext || !sAppContext->mDevice) {
+        return;
+    }
     sAppContext->mDevice->setHead(aX, aY, aZ, aW);
 }
 
+JNI_METHOD(void, setControllerOrientation)
+(JNIEnv*, jobject, jdouble aX, jdouble aY, jdouble aZ, jdouble aW) {
+    if (!sAppContext || !sAppContext->mDevice) {
+        return;
+    }
+    sAppContext->mDevice->setControllerOrientation(aX, aY, aZ, aW);
+}
+
 jint JNI_OnLoad(JavaVM* aVm, void*) {
     if (sAppContext) {
         return JNI_VERSION_1_6;
diff --git a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java
index 9d08749227c..2d3aa1541b2 100644
--- a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java
+++ b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java
@@ -6,25 +6,39 @@
 package com.igalia.wolvic;
 
 import androidx.activity.ComponentActivity;
+
+import androidx.annotation.Keep;
+
+import android.app.Presentation;
+import android.content.Context;
+import android.content.DialogInterface;
 import android.content.Intent;
+import android.content.res.Resources;
+import android.hardware.display.DisplayManager;
+import android.hardware.Sensor;
+import android.hardware.SensorEvent;
+import android.hardware.SensorEventListener;
+import android.hardware.SensorManager;
 import android.hardware.usb.UsbDevice;
 import android.opengl.GLSurfaceView;
 import android.os.Build;
 import android.os.Bundle;
+import android.util.DisplayMetrics;
 import android.util.Log;
+import android.view.Display;
 import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.WindowInsets;
 import android.view.WindowInsetsController;
+import android.view.WindowManager;
+import android.widget.Button;
+import android.widget.RelativeLayout;
 import android.widget.TextView;
 
-import androidx.annotation.Keep;
-
 import com.huawei.usblib.DisplayMode;
 import com.huawei.usblib.DisplayModeCallback;
 import com.huawei.usblib.VisionGlass;
-
 import com.igalia.wolvic.utils.SystemUtils;
 
 import java.util.ArrayList;
@@ -33,8 +47,11 @@
 import javax.microedition.khronos.egl.EGLConfig;
 import javax.microedition.khronos.opengles.GL10;
 
-public class PlatformActivity extends ComponentActivity {
+public class PlatformActivity extends ComponentActivity implements SensorEventListener {
     static String LOGTAG = SystemUtils.createLogtag(PlatformActivity.class);
+    private DisplayManager mDisplayManager;
+    private Display mPresentationDisplay;
+    private VisionGlassPresentation mActivePresentation;
 
     @SuppressWarnings("unused")
     public static boolean filterPermission(final String aPermission) {
@@ -55,12 +72,8 @@ protected Intent getStoreIntent() {
         return null;
     }
 
-    private GLSurfaceView mView;
-    private TextView mFrameRate;
     private final ArrayList<Runnable> mPendingEvents = new ArrayList<>();
-    private boolean mSurfaceCreated = false;
-    private int mFrameCount;
-    private long mLastFrameTime = System.currentTimeMillis();
+    private SensorManager mSensorManager;
 
     final Object mRenderLock = new Object();
 
@@ -77,7 +90,6 @@ protected Intent getStoreIntent() {
             mRenderLock.notifyAll();
         }
     };
-    private int count = 0;
     private final Runnable activityResumedRunnable = this::activityResumed;
 
     @Override
@@ -87,25 +99,27 @@ protected void onCreate(Bundle savedInstanceState) {
 
         VisionGlass.getInstance().init(getApplication());
 
+        mDisplayManager = (DisplayManager) getSystemService(Context.DISPLAY_SERVICE);
         boolean wasImuStarted = false;
         boolean isAskingForPermission = false;
         do {
             if (VisionGlass.getInstance().isConnected()) {
-                if (isAskingForPermission) {
-                    try {
-                        Thread.sleep(100);
-                    } catch (InterruptedException e) {
-                        e.printStackTrace();
-                    }
-                }
                 if (VisionGlass.getInstance().hasUsbPermission()) {
                     Log.d(LOGTAG, "Device has USB permission -> registering callback for startImu");
                     wasImuStarted = true;
                     VisionGlass.getInstance().startImu((w, x, y, z) -> queueRunnable(() -> setHead(x, y, z, w)));
                 } else {
-                    Log.w(LOGTAG, "Device does not have USB permission -> asking");
-                    VisionGlass.getInstance().requestUsbPermission();
-                    isAskingForPermission = true;
+                    if (isAskingForPermission) {
+                        try {
+                            Thread.sleep(100);
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                    } else {
+                        Log.w(LOGTAG, "Device does not have USB permission -> asking");
+                        VisionGlass.getInstance().requestUsbPermission();
+                        isAskingForPermission = true;
+                    }
                 }
             } else {
                 // TODO: show a dialog asking the user to put on the glasses
@@ -113,82 +127,92 @@ protected void onCreate(Bundle savedInstanceState) {
             }
         } while (!wasImuStarted);
 
+
         VisionGlass.getInstance().setDisplayMode(DisplayMode.vr2d, new DisplayModeCallback() {
             @Override
             public void onSuccess(DisplayMode displayMode) { Log.d(LOGTAG, "Successfully switched to 2D mode"); }
 
             @Override
-            public void onError(String s, int i) { Log.d(LOGTAG, "Error " + i + " failed to switch to 2D mode " + s); }
+            public void onError(String s, int i) {
+                Log.d(LOGTAG, "Error " + i + " failed to switch to 2D mode " + s);
+            }
         });
 
+        mSensorManager = (SensorManager) getSystemService(Context.SENSOR_SERVICE);
+
         setContentView(R.layout.visionglass_layout);
-        mFrameRate = findViewById(R.id.frame_rate_text);
-        mView = findViewById(R.id.gl_view);
-        mView.setEGLContextClientVersion(3);
-        mView.setEGLConfigChooser(8, 8, 8, 0, 16, 0);
-        mView.setPreserveEGLContextOnPause(true);
-
-        mView.setRenderer(
-                new GLSurfaceView.Renderer() {
-                    @Override
-                    public void onSurfaceCreated(GL10 gl, EGLConfig config) {
-                        Log.d(LOGTAG, "In onSurfaceCreated");
-                        activityCreated(getAssets());
-                        mSurfaceCreated = true;
-                        notifyPendingEvents();
-                    }
 
-                    @Override
-                    public void onSurfaceChanged(GL10 gl, int width, int height) {
-                        Log.d(LOGTAG, "In onSurfaceChanged");
-                        // There is no way to actually get these values from the device so we have
-                        // to hardcode them. The width and height we get here are from device's
-                        // screen (the phone display), not the glasses' screen.
-                        final int VISION_GLASS_WIDTH = 3840;
-                        final int VISION_GLASS_HEIGHT = 1080;
-                        updateViewport(VISION_GLASS_WIDTH, VISION_GLASS_HEIGHT);
-                    }
+        View touchpad = findViewById(R.id.touchpad);
+        // Make touchpad square.
+        RelativeLayout.LayoutParams layoutParams = (RelativeLayout.LayoutParams) touchpad.getLayoutParams();
+        layoutParams.height = getResources().getDisplayMetrics().widthPixels;
+        touchpad.setLayoutParams(layoutParams);
 
-                    @Override
-                    public void onDrawFrame(GL10 gl) {
-                        mFrameCount++;
-                        long ctime = System.currentTimeMillis();
-                        if ((ctime - mLastFrameTime) >= 1000) {
-                            final int value =  Math.round(mFrameCount / ((ctime - mLastFrameTime) / 1000.0f));
-                            mLastFrameTime = ctime;
-                            mFrameCount = 0;
-                            runOnUiThread(() -> mFrameRate.setText(String.valueOf(value)));
-                        }
-                        drawGL();
-                    }
-                });
+        touchpad.setOnClickListener((View.OnClickListener) v -> {
+            // We don't really need the coordinates of the click because we use the position
+            // of the aim in the 3D environment.
+            queueRunnable(() -> touchEvent(false, 0, 0));
+        });
+
+        touchpad.setOnTouchListener((view, event) -> {
+            switch (event.getAction()) {
+                case MotionEvent.ACTION_DOWN:
+                case MotionEvent.ACTION_MOVE:
+                    // We don't really need the coordinates of the click because we use the position
+                    // of the aim in the 3D environment.
+                    queueRunnable(() -> touchEvent(true, 0, 0));
+                    break;
+                case MotionEvent.ACTION_UP:
+                    // We'd emit the touchEvent in the onClick listener of the view. This way both
+                    // user and system activated clicks (e.g. a11y) will work.
+                    view.performClick();
+                    break;
+                default:
+                    return false;
+            }
+            return true;
+        });
+
+        // Find the buttons by their id
+        Button backButton = findViewById(R.id.back_button);
+        Button homeButton = findViewById(R.id.home_button);
+
+        // Set click listeners for the buttons
+        backButton.setOnClickListener(new View.OnClickListener() {
+            @Override
+            public void onClick(View v) {
+                onBackPressed();
+            }
+        });
+
+        homeButton.setOnClickListener(new View.OnClickListener() {
+            @Override
+            public void onClick(View v) {
+                // Handle home button click
+                Log.d(LOGTAG, "Home button clicked");
+            }
+        });
+
+        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
     }
 
+    // SensorEventListener overrides
     @Override
-    public boolean onTouchEvent(MotionEvent aEvent) {
-        if (aEvent.getActionIndex() != 0) {
-            Log.e(LOGTAG,"aEvent.getActionIndex()=" + aEvent.getActionIndex());
-            return false;
-        }
-
-        int action = aEvent.getAction();
-        boolean down;
-        if (action == MotionEvent.ACTION_DOWN) {
-            down = true;
-        } else if (action == MotionEvent.ACTION_UP) {
-            down = false;
-        } else if (action == MotionEvent.ACTION_MOVE) {
-            down = true;
-        } else {
-            return false;
-        }
+    public void onSensorChanged(SensorEvent event) {
+        // retrieve the device orientation from sensorevent in the form of quaternion
+        if (event.sensor.getType() != Sensor.TYPE_GAME_ROTATION_VECTOR)
+            return;
+
+        float[] quaternion = new float[4];
+        SensorManager.getQuaternionFromVector(quaternion, event.values);
+        // The quaternion is returned in the form [w, x, z, y] but we use it as [x, y, z, w].
+        // See https://developer.android.com/reference/android/hardware/Sensor#TYPE_ROTATION_VECTOR
+        queueRunnable(() -> setControllerOrientation(quaternion[1], quaternion[3], quaternion[2], quaternion[0]));
+    }
 
-        final boolean isDown = down;
+    @Override
+    public void onAccuracyChanged(Sensor sensor, int accuracy) {
 
-        final float xx = aEvent.getX(0);
-        final float yy = aEvent.getY(0);
-        queueRunnable(() -> touchEvent(isDown, xx, yy));
-        return true;
     }
 
     @Override
@@ -211,6 +235,7 @@ public boolean onGenericMotionEvent(MotionEvent aEvent) {
     @Override
     protected void onPause() {
         Log.d(LOGTAG, "PlatformActivity onPause");
+        super.onPause();
         synchronized (mRenderLock) {
             queueRunnable(activityPausedRunnable);
             try {
@@ -219,17 +244,31 @@ protected void onPause() {
                 Log.e(LOGTAG, "activityPausedRunnable interrupted: " + e.toString());
             }
         }
-        mView.onPause();
-        super.onPause();
+        if (mActivePresentation != null)
+            mActivePresentation.mGLView.onPause();
+
+        // Unregister from the display manager.
+        mDisplayManager.unregisterDisplayListener(mDisplayListener);
+        mSensorManager.unregisterListener(this);
     }
 
     @Override
     protected void onResume() {
         Log.d(LOGTAG, "PlatformActivity onResume");
         super.onResume();
-        mView.onResume();
+
+        updateDisplays();
+        showPresentation();
+
+        if (mActivePresentation != null && mActivePresentation.mGLView != null)
+            mActivePresentation.mGLView.onResume();
+
         queueRunnable(activityResumedRunnable);
         setImmersiveSticky();
+
+        // Register to receive events from the display manager.
+        mDisplayManager.registerDisplayListener(mDisplayListener, null);
+        mSensorManager.registerListener(this, mSensorManager.getDefaultSensor(Sensor.TYPE_GAME_ROTATION_VECTOR), SensorManager.SENSOR_DELAY_NORMAL);
     }
 
     @Override
@@ -268,13 +307,13 @@ void setImmersiveSticky() {
     }
 
     void queueRunnable(Runnable aRunnable) {
-        if (mSurfaceCreated) {
-            mView.queueEvent(aRunnable);
+        if (mActivePresentation != null) {
+            mActivePresentation.mGLView.queueEvent(aRunnable);
         } else {
             synchronized (mPendingEvents) {
                 mPendingEvents.add(aRunnable);
             }
-            if (mSurfaceCreated) {
+            if (mActivePresentation != null) {
                 notifyPendingEvents();
             }
         }
@@ -283,12 +322,124 @@ void queueRunnable(Runnable aRunnable) {
     private void notifyPendingEvents() {
         synchronized (mPendingEvents) {
             for (Runnable runnable: mPendingEvents) {
-                mView.queueEvent(runnable);
+                mActivePresentation.mGLView.queueEvent(runnable);
             }
             mPendingEvents.clear();
         }
     }
 
+    private void updateDisplays() {
+        Display[] displays = mDisplayManager.getDisplays(DisplayManager.DISPLAY_CATEGORY_PRESENTATION);
+        if (displays.length == 0) {
+            mPresentationDisplay = null;
+            return;
+        }
+
+        mPresentationDisplay = displays[0];
+    }
+
+    private final DisplayManager.DisplayListener mDisplayListener =
+            new DisplayManager.DisplayListener() {
+                @Override
+                public void onDisplayAdded(int displayId) {
+                    updateDisplays();
+                }
+
+                @Override
+                public void onDisplayChanged(int displayId) {
+                    updateDisplays();
+                }
+
+                @Override
+                public void onDisplayRemoved(int displayId) {
+                    updateDisplays();
+                }
+            };
+
+    private final DialogInterface.OnDismissListener mOnDismissListener =
+            new DialogInterface.OnDismissListener() {
+                @Override
+                public void onDismiss(DialogInterface dialog) {
+                    mActivePresentation = null;
+                }
+            };
+
+    private void showPresentation() {
+        if (mActivePresentation != null) {
+            return;
+        }
+        if (mPresentationDisplay == null) {
+            Log.e(LOGTAG, "No suitable displays found");
+            return;
+        }
+        VisionGlassPresentation presentation = new VisionGlassPresentation(this, mPresentationDisplay);
+        Display.Mode [] modes = mPresentationDisplay.getSupportedModes();
+        presentation.setPreferredDisplayMode(modes[0].getModeId());
+        presentation.show();
+        presentation.setOnDismissListener(mOnDismissListener);
+        mActivePresentation = presentation;
+    }
+
+    private final class VisionGlassPresentation extends Presentation {
+
+        private GLSurfaceView mGLView;
+
+        public VisionGlassPresentation(Context context, Display display) {
+            super(context, display);
+        }
+
+        /**
+         * Sets the preferred display mode id for the presentation.
+         */
+        public void setPreferredDisplayMode(int modeId) {
+            WindowManager.LayoutParams params = getWindow().getAttributes();
+            params.preferredDisplayModeId = modeId;
+            getWindow().setAttributes(params);
+        }
+
+        @Override
+        protected void onCreate(Bundle savedInstanceState) {
+            // Be sure to call the super class.
+            super.onCreate(savedInstanceState);
+
+            // Get the resources for the context of the presentation.
+            // Notice that we are getting the resources from the context of the presentation.
+            Resources r = getContext().getResources();
+
+            // Inflate the layout.
+            setContentView(R.layout.visionglass_presentation_layout);
+
+            mGLView = findViewById(R.id.gl_presentation_view);
+            mGLView.setEGLContextClientVersion(3);
+            mGLView.setEGLConfigChooser(8, 8, 8, 0, 16, 0);
+            mGLView.setPreserveEGLContextOnPause(true);
+
+            mGLView.setRenderer(new GLSurfaceView.Renderer() {
+                @Override
+                public void onSurfaceCreated(GL10 gl, EGLConfig config) {
+                    activityCreated(getAssets());
+                    notifyPendingEvents();
+                }
+
+                @Override
+                public void onSurfaceChanged(GL10 gl, int width, int height) {
+                    updateViewport(width, height);
+                }
+
+                @Override
+                public void onDrawFrame(GL10 gl) {
+                    drawGL();
+                }
+            });
+        }
+    }
+
+    @Keep
+    @SuppressWarnings("unused")
+    private void setRenderMode(final int aMode) {
+        runOnUiThread(() -> setImmersiveSticky());
+    }
+
     private native void activityCreated(Object aAssetManager);
     private native void updateViewport(int width, int height);
     private native void activityPaused();
@@ -297,4 +448,5 @@ private void notifyPendingEvents() {
     private native void drawGL();
     private native void touchEvent(boolean aDown, float aX, float aY);
     private native void setHead(double x, double y, double z, double w);
+    private native void setControllerOrientation(double x, double y, double z, double w);
 }