Skeletal Summary 2: Electric Boogaloo

src/input.rs

@@ -275,10 +275,7 @@ enum ActionData {
         last_value: (AtomicF32, AtomicF32),
     },
     Pose,
-    Skeleton {
-        hand: Hand,
-        hand_tracker: Option<xr::HandTracker>,
-    },
+    Skeleton(Hand),
     Haptic(xr::Action<xr::Haptic>),
 }
 
@@ -530,17 +527,21 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
     fn GetSkeletalSummaryData(
         &self,
         action: vr::VRActionHandle_t,
-        _: vr::EVRSummaryType,
+        summary_type: vr::EVRSummaryType,
         data: *mut vr::VRSkeletalSummaryData_t,
     ) -> vr::EVRInputError {
-        crate::warn_unimplemented!("GetSkeletalSummaryData");
-        get_action_from_handle!(self, action, session_data, _action);
-        unsafe {
-            data.write(vr::VRSkeletalSummaryData_t {
-                flFingerSplay: [0.2; 4],
-                flFingerCurl: [0.0; 5],
-            })
-        }
+        get_action_from_handle!(self, action, session_data, action);
+
+        let ActionData::Skeleton(hand) = action else {
+            return vr::EVRInputError::WrongType;
+        };
+
+        let Some(data) = (unsafe { data.as_mut() }) else {
+            return vr::EVRInputError::InvalidParam;
+        };
+
+        self.get_bone_summary_from_hand_tracking(&session_data, summary_type, data, *hand);
+
         vr::EVRInputError::None
     }
     fn GetSkeletalBoneData(
@@ -559,22 +560,11 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
         };
 
         get_action_from_handle!(self, handle, session_data, action);
-        let ActionData::Skeleton { hand, hand_tracker } = action else {
+        let ActionData::Skeleton(hand) = action else {
             return vr::EVRInputError::WrongType;
         };
 
-        if let Some(hand_tracker) = hand_tracker.as_ref() {
-            self.get_bones_from_hand_tracking(
-                &session_data,
-                transform_space,
-                hand_tracker,
-                *hand,
-                transforms,
-            )
-        } else {
-            self.get_estimated_bones(&session_data, transform_space, *hand, transforms);
-        }
-
+        self.get_bones_from_hand_tracking(&session_data, transform_space, *hand, transforms);
         vr::EVRInputError::None
     }
     fn GetSkeletalTrackingLevel(
@@ -583,7 +573,7 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
         level: *mut vr::EVRSkeletalTrackingLevel,
     ) -> vr::EVRInputError {
         get_action_from_handle!(self, action, data, action);
-        let ActionData::Skeleton { hand, .. } = action else {
+        let ActionData::Skeleton(hand) = action else {
             return vr::EVRInputError::WrongType;
         };
 
@@ -627,7 +617,7 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
         };
 
         get_action_from_handle!(self, handle, session_data, action);
-        let ActionData::Skeleton { hand, .. } = action else {
+        let ActionData::Skeleton(hand) = action else {
             return vr::EVRInputError::WrongType;
         };
 
@@ -688,7 +678,7 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
             return vr::EVRInputError::InvalidHandle;
         };
         let origin = match loaded.try_get_action(action) {
-            Ok(ActionData::Skeleton { hand, .. }) => match hand {
+            Ok(ActionData::Skeleton(hand)) => match hand {
                 Hand::Left => self.left_hand_key.data().as_ffi(),
                 Hand::Right => self.right_hand_key.data().as_ffi(),
             },
@@ -823,7 +813,7 @@ impl<C: openxr_data::Compositor> vr::IVRInput010_Interface for Input<C> {
                     }
                 }
             }
-            Ok(ActionData::Skeleton { hand, .. }) => {
+            Ok(ActionData::Skeleton(hand)) => {
                 if subaction_path != xr::Path::NULL {
                     return vr::EVRInputError::InvalidDevice;
                 }
@@ -1372,8 +1362,25 @@ impl<C: openxr_data::Compositor> Input<C> {
                 if let Some(controller) = controller.as_mut() {
                     controller.interaction_profile = Some(p);
                 } else {
+                    let hand_tracker = session_data
+                        .session
+                        .create_hand_tracker(hand.into())
+                        .inspect_err(|e| {
+                            if !matches!(
+                                *e,
+                                xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT
+                                    | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED
+                            ) {
+                                log::warn!("Failed to create hand tracker for hand {hand:?}: {e}");
+                            }
+                        })
+                        .ok();
                     devices_to_create.push((
-                        TrackedDeviceType::Controller { hand },
+                        TrackedDeviceType::Controller {
+                            hand,
+                            hand_tracker,
+                            skeleton_cache: Mutex::new(Default::default()),
+                        },
                         Some(profile_path),
                         Some(p),
                     ));

src/input/action_manifest.rs

@@ -104,7 +104,6 @@ impl<C: openxr_data::Compositor> Input<C> {
 
         let actions = load_actions(
             &self.openxr.instance,
-            &session_data.session,
             english.as_ref(),
             &mut sets,
             manifest.actions,
@@ -441,7 +440,6 @@ fn create_action<T: xr::ActionTy>(
 type LoadedActionDataMap = HashMap<String, super::ActionData>;
 fn load_actions(
     instance: &xr::Instance,
-    session: &xr::Session<xr::AnyGraphics>,
     english: Option<&Localization>,
     sets: &mut HashMap<String, xr::ActionSet>,
     actions: Vec<ActionType>,
@@ -485,25 +483,7 @@ fn load_actions(
             ActionType::Pose(data) => (&data.name, Pose),
             ActionType::Skeleton(SkeletonData { skeleton, data }) => {
                 trace!("Creating skeleton action {}", data.name.path);
-                let hand_tracker = match session.create_hand_tracker(match skeleton {
-                    Hand::Left => xr::Hand::LEFT,
-                    Hand::Right => xr::Hand::RIGHT,
-                }) {
-                    Ok(t) => Some(t),
-                    Err(
-                        xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT
-                        | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED,
-                    ) => None,
-                    Err(other) => panic!("Creating hand tracker failed: {other:?}"),
-                };
-
-                (
-                    &data.name,
-                    Skeleton {
-                        hand: *skeleton,
-                        hand_tracker,
-                    },
-                )
+                (&data.name, Skeleton(*skeleton))
             }
             ActionType::Vibration(data) => (&data.name, Haptic(create_action!(xr::Haptic, data))),
         };
@@ -1503,7 +1483,7 @@ fn handle_skeleton_bindings(
         };
 
         match &context.actions[&output.path] {
-            super::ActionData::Skeleton { hand, .. } => {
+            super::ActionData::Skeleton(hand) => {
                 let bound_hand = match path.as_str() {
                     "/user/hand/left/input/skeleton/left" => Hand::Left,
                     "/user/hand/right/input/skeleton/right" => Hand::Right,

src/input/devices.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::ffi::{CStr, CString};
 use std::sync::Mutex;
 
@@ -19,6 +20,8 @@ pub enum TrackedDeviceType {
     Hmd,
     Controller {
         hand: Hand,
+        hand_tracker: Option<xr::HandTracker>,
+        skeleton_cache: Mutex<HashMap<u64, Option<xr::HandJointLocations>>>,
     },
     #[cfg(feature = "monado")]
     GenericTracker {
@@ -31,7 +34,7 @@ impl std::fmt::Debug for TrackedDeviceType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             TrackedDeviceType::Hmd => write!(f, "HMD"),
-            TrackedDeviceType::Controller { hand } => write!(f, "Controller ({:?})", hand),
+            TrackedDeviceType::Controller { hand, .. } => write!(f, "Controller ({:?})", hand),
             #[cfg(feature = "monado")]
             TrackedDeviceType::GenericTracker { serial, .. } => {
                 write!(f, "Generic Tracker ({})", serial.to_string_lossy())
@@ -158,8 +161,36 @@ impl TrackedDevice {
         *pose_cache
     }
 
+    pub fn get_hand_skeleton(
+        &self,
+        xr_data: &OpenXrData<impl crate::openxr_data::Compositor>,
+        base: &xr::Space,
+    ) -> Option<xr::HandJointLocations> {
+        let TrackedDeviceType::Controller {
+            hand_tracker,
+            skeleton_cache,
+            ..
+        } = self.get_type()
+        else {
+            return None;
+        };
+        let mut skeleton_cache = skeleton_cache.lock().unwrap();
+        if let Some(skeleton) = skeleton_cache.get(&base.as_raw().into_raw()) {
+            return *skeleton;
+        }
+
+        let joints = base
+            .locate_hand_joints(hand_tracker.as_ref()?, xr_data.display_time.get())
+            .unwrap_or_default();
+        skeleton_cache.insert(base.as_raw().into_raw(), joints);
+        joints
+    }
+
     pub fn clear_pose_cache(&self) {
         std::mem::take(&mut *self.pose_cache.lock().unwrap());
+        if let TrackedDeviceType::Controller { skeleton_cache, .. } = self.get_type() {
+            skeleton_cache.lock().unwrap().clear();
+        }
     }
 
     pub fn has_connected_changed(&mut self) -> bool {
@@ -184,7 +215,7 @@ impl TrackedDevice {
 
     fn get_string_property(&self, property: vr::ETrackedDeviceProperty) -> Option<&CStr> {
         let hand = match self.device_type {
-            TrackedDeviceType::Controller { hand } => hand,
+            TrackedDeviceType::Controller { hand, .. } => hand,
             _ => Hand::Left,
         };
 

src/input/profiles/knuckles.rs

@@ -144,6 +144,8 @@ impl InteractionProfile for Knuckles {
                 .leftright("input/thumbstick/touch")
                 .into_iter()
                 .chain(stp.leftright("input/trackpad/touch"))
+                .chain(stp.leftright("input/a/touch"))
+                .chain(stp.leftright("input/b/touch"))
                 .collect(),
             index_touch: stp.leftright("input/trigger/touch"),
             index_curl: stp.leftright("input/trigger/value"),