[OpenXR] Add support for XR_FB_keyboard_tracking

Only Quest devices support this at the moment. Tested on Quest2.

The bulk of the implementation was added to OpenXRInput, bacause
OpenXRInputSource is meant for controllers and DeviceDelegateOpenXR
is already bloated. In the future, we might want to move this to
OpenXRTrackedKeyboard if code gets more complex.

Note that this patch doesn't deal with rendering of the keyboard, nor
the hands-on-keyboard passthrough functionality. It just track the
keyboard's id, availability and location on every frame. A subsequent
series will add the rest of the functionality to make this usable,
namely, XR_FB_render_mode and XR_FB_passthrough_keyboard_hands
extensions support.

app/src/openxr/cpp/DeviceDelegateOpenXR.cpp

@@ -1530,6 +1530,7 @@ DeviceDelegateOpenXR::EnterVR(const crow::BrowserEGLContext& aEGLContext) {
     m.controllersReadyCallback();
     m.controllersReadyCallback = nullptr;
   }
+  m.input->SetKeyboardTrackingEnabled(m.keyboardTrackingSupported);
 
   vrb::RenderContextPtr context = m.context.lock();
 

app/src/openxr/cpp/OpenXRInput.cpp

@@ -2,6 +2,7 @@
 #include "OpenXRHelpers.h"
 #include "OpenXRInputSource.h"
 #include "OpenXRActionSet.h"
+#include "OpenXRExtensions.h"
 #include <vector>
 
 namespace crow {
@@ -80,6 +81,10 @@ XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace,
     input->Update(frameState, baseSpace, head, offsets, renderMode, delegate);
   }
 
+  // Update tracked keyboard
+  if (keyboardTrackingFB != nullptr)
+    UpdateTrackedKeyboard(frameState, baseSpace);
+
   return XR_SUCCESS;
 }
 
@@ -142,7 +147,70 @@ HandMeshBufferPtr OpenXRInput::GetNextHandMeshBuffer(const int32_t aControllerIn
   return mInputSources.at(aControllerIndex)->GetNextHandMeshBuffer();
 }
 
+void OpenXRInput::UpdateTrackedKeyboard(const XrFrameState& frameState, XrSpace baseSpace) {
+  CHECK(keyboardTrackingFB != nullptr);
+  CHECK(OpenXRExtensions::xrQuerySystemTrackedKeyboardFB != nullptr);
+  CHECK(OpenXRExtensions::xrCreateKeyboardSpaceFB != nullptr);
+
+  XrKeyboardTrackingQueryFB queryInfo = {
+    .type = XR_TYPE_KEYBOARD_TRACKING_QUERY_FB,
+    .flags = XR_KEYBOARD_TRACKING_QUERY_LOCAL_BIT_FB,
+  };
+  XrKeyboardTrackingDescriptionFB kbdDesc;
+  CHECK_XRCMD(OpenXRExtensions::xrQuerySystemTrackedKeyboardFB(mSession, &queryInfo, &kbdDesc));
+
+  // Check if keyboard disappeared or changed, and clear up state if so
+  if ((kbdDesc.flags & XR_KEYBOARD_TRACKING_EXISTS_BIT_FB) == 0 ||
+       kbdDesc.trackedKeyboardId != keyboardTrackingFB->trackedKeyboardId) {
+    if (keyboardTrackingFB->space != XR_NULL_HANDLE) {
+      xrDestroySpace(keyboardTrackingFB->space);
+      keyboardTrackingFB->space = XR_NULL_HANDLE;
+    }
+    keyboardTrackingFB->trackedKeyboardId = 0;
+  }
+
+  // Bail-out if no keyboard exists
+  if ((kbdDesc.flags & XR_KEYBOARD_TRACKING_EXISTS_BIT_FB) == 0)
+    return;
+
+  keyboardTrackingFB->trackedKeyboardId = kbdDesc.trackedKeyboardId;
+
+  // Create the XrSpace to track the keyboard, if not done already
+  if (keyboardTrackingFB->space == XR_NULL_HANDLE) {
+    VRB_LOG("Tracked keyboard detected: %s", kbdDesc.name);
+
+    XrKeyboardSpaceCreateInfoFB createInfo{
+      .type = XR_TYPE_KEYBOARD_SPACE_CREATE_INFO_FB,
+      .trackedKeyboardId = kbdDesc.trackedKeyboardId,
+    };
+    CHECK_XRCMD(OpenXRExtensions::xrCreateKeyboardSpaceFB(mSession, &createInfo,
+                                                          &keyboardTrackingFB->space));
+  }
+
+  // If keyboard is active, query and store the keyboard's pose
+  if (kbdDesc.flags & XR_KEYBOARD_TRACKING_CONNECTED_BIT_FB) {
+    CHECK(keyboardTrackingFB->space != XR_NULL_HANDLE);
+
+    keyboardTrackingFB->location.type = XR_TYPE_SPACE_LOCATION;
+    CHECK_XRCMD(xrLocateSpace(keyboardTrackingFB->space, baseSpace, frameState.predictedDisplayTime,
+                              &keyboardTrackingFB->location));
+  }
+}
+
+void OpenXRInput::SetKeyboardTrackingEnabled(bool enabled) {
+  if (enabled && keyboardTrackingFB == nullptr) {
+    keyboardTrackingFB = std::make_unique<KeyboardTrackingFB>();
+  } else if (!enabled && keyboardTrackingFB != nullptr) {
+    keyboardTrackingFB.reset();
+    keyboardTrackingFB = nullptr;
+  }
+}
+
 OpenXRInput::~OpenXRInput() {
+  if (keyboardTrackingFB != nullptr) {
+    keyboardTrackingFB.reset();
+    keyboardTrackingFB = nullptr;
+  }
 }
 
 } // namespace crow

app/src/openxr/cpp/OpenXRInput.h

@@ -23,7 +23,19 @@ typedef std::shared_ptr<HandMeshBuffer> HandMeshBufferPtr;
 
 class OpenXRInput {
 private:
+  struct KeyboardTrackingFB {
+    uint64_t trackedKeyboardId { 0 };
+    XrSpace space = XR_NULL_HANDLE;
+    XrSpaceLocation location;
+    ~KeyboardTrackingFB() {
+      if (space != XR_NULL_HANDLE)
+        xrDestroySpace(space);
+    }
+  };
+  typedef std::unique_ptr<KeyboardTrackingFB> KeyboardTrackingFBPtr;
+
   OpenXRInput(XrInstance, XrSession, XrSystemProperties, ControllerDelegate& delegate);
+  void UpdateTrackedKeyboard(const XrFrameState& frameState, XrSpace baseSpace);
 
   OpenXRInputMapping* GetActiveInputMapping() const;
 
@@ -32,6 +44,7 @@ class OpenXRInput {
   XrSystemProperties mSystemProperties;
   std::vector<OpenXRInputSourcePtr> mInputSources;
   OpenXRActionSetPtr mActionSet;
+  KeyboardTrackingFBPtr keyboardTrackingFB { nullptr };
 
 public:
   static OpenXRInputPtr Create(XrInstance, XrSession, XrSystemProperties, ControllerDelegate& delegate);
@@ -43,6 +56,7 @@ class OpenXRInput {
   bool AreControllersReady() const;
   void SetHandMeshBufferSizes(const uint32_t indexCount, const uint32_t vertexCount);
   HandMeshBufferPtr GetNextHandMeshBuffer(const int32_t aControllerIndex);
+  void SetKeyboardTrackingEnabled(bool enabled);
   ~OpenXRInput();
 };