diff --git a/src/input.rs b/src/input.rs index c5a71fc0..99d1bd43 100644 --- a/src/input.rs +++ b/src/input.rs @@ -275,10 +275,7 @@ enum ActionData { last_value: (AtomicF32, AtomicF32), }, Pose, - Skeleton { - hand: Hand, - hand_tracker: Option, - }, + Skeleton(Hand), Haptic(xr::Action), } @@ -530,17 +527,21 @@ impl vr::IVRInput010_Interface for Input { fn GetSkeletalSummaryData( &self, action: vr::VRActionHandle_t, - _: vr::EVRSummaryType, + summary_type: vr::EVRSummaryType, data: *mut vr::VRSkeletalSummaryData_t, ) -> vr::EVRInputError { - crate::warn_unimplemented!("GetSkeletalSummaryData"); - get_action_from_handle!(self, action, session_data, _action); - unsafe { - data.write(vr::VRSkeletalSummaryData_t { - flFingerSplay: [0.2; 4], - flFingerCurl: [0.0; 5], - }) - } + get_action_from_handle!(self, action, session_data, action); + + let ActionData::Skeleton(hand) = action else { + return vr::EVRInputError::WrongType; + }; + + let Some(data) = (unsafe { data.as_mut() }) else { + return vr::EVRInputError::InvalidParam; + }; + + self.get_bone_summary_from_hand_tracking(&session_data, summary_type, data, *hand); + vr::EVRInputError::None } fn GetSkeletalBoneData( @@ -559,22 +560,11 @@ impl vr::IVRInput010_Interface for Input { }; get_action_from_handle!(self, handle, session_data, action); - let ActionData::Skeleton { hand, hand_tracker } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; - if let Some(hand_tracker) = hand_tracker.as_ref() { - self.get_bones_from_hand_tracking( - &session_data, - transform_space, - hand_tracker, - *hand, - transforms, - ) - } else { - self.get_estimated_bones(&session_data, transform_space, *hand, transforms); - } - + self.get_bones_from_hand_tracking(&session_data, transform_space, *hand, transforms); vr::EVRInputError::None } fn GetSkeletalTrackingLevel( @@ -583,7 +573,7 @@ impl vr::IVRInput010_Interface for Input { level: *mut vr::EVRSkeletalTrackingLevel, ) -> vr::EVRInputError { get_action_from_handle!(self, action, data, action); - let ActionData::Skeleton { hand, .. } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; @@ -627,7 +617,7 @@ impl vr::IVRInput010_Interface for Input { }; get_action_from_handle!(self, handle, session_data, action); - let ActionData::Skeleton { hand, .. } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; @@ -688,7 +678,7 @@ impl vr::IVRInput010_Interface for Input { return vr::EVRInputError::InvalidHandle; }; let origin = match loaded.try_get_action(action) { - Ok(ActionData::Skeleton { hand, .. }) => match hand { + Ok(ActionData::Skeleton(hand)) => match hand { Hand::Left => self.left_hand_key.data().as_ffi(), Hand::Right => self.right_hand_key.data().as_ffi(), }, @@ -823,7 +813,7 @@ impl vr::IVRInput010_Interface for Input { } } } - Ok(ActionData::Skeleton { hand, .. }) => { + Ok(ActionData::Skeleton(hand)) => { if subaction_path != xr::Path::NULL { return vr::EVRInputError::InvalidDevice; } @@ -1372,8 +1362,25 @@ impl Input { if let Some(controller) = controller.as_mut() { controller.interaction_profile = Some(p); } else { + let hand_tracker = session_data + .session + .create_hand_tracker(hand.into()) + .inspect_err(|e| { + if !matches!( + *e, + xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT + | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED + ) { + log::warn!("Failed to create hand tracker for hand {hand:?}: {e}"); + } + }) + .ok(); devices_to_create.push(( - TrackedDeviceType::Controller { hand }, + TrackedDeviceType::Controller { + hand, + hand_tracker, + skeleton_cache: Mutex::new(Default::default()), + }, Some(profile_path), Some(p), )); diff --git a/src/input/action_manifest.rs b/src/input/action_manifest.rs index 8ed9894d..68c31878 100644 --- a/src/input/action_manifest.rs +++ b/src/input/action_manifest.rs @@ -104,7 +104,6 @@ impl Input { let actions = load_actions( &self.openxr.instance, - &session_data.session, english.as_ref(), &mut sets, manifest.actions, @@ -441,7 +440,6 @@ fn create_action( type LoadedActionDataMap = HashMap; fn load_actions( instance: &xr::Instance, - session: &xr::Session, english: Option<&Localization>, sets: &mut HashMap, actions: Vec, @@ -485,25 +483,7 @@ fn load_actions( ActionType::Pose(data) => (&data.name, Pose), ActionType::Skeleton(SkeletonData { skeleton, data }) => { trace!("Creating skeleton action {}", data.name.path); - let hand_tracker = match session.create_hand_tracker(match skeleton { - Hand::Left => xr::Hand::LEFT, - Hand::Right => xr::Hand::RIGHT, - }) { - Ok(t) => Some(t), - Err( - xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT - | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED, - ) => None, - Err(other) => panic!("Creating hand tracker failed: {other:?}"), - }; - - ( - &data.name, - Skeleton { - hand: *skeleton, - hand_tracker, - }, - ) + (&data.name, Skeleton(*skeleton)) } ActionType::Vibration(data) => (&data.name, Haptic(create_action!(xr::Haptic, data))), }; @@ -1503,7 +1483,7 @@ fn handle_skeleton_bindings( }; match &context.actions[&output.path] { - super::ActionData::Skeleton { hand, .. } => { + super::ActionData::Skeleton(hand) => { let bound_hand = match path.as_str() { "/user/hand/left/input/skeleton/left" => Hand::Left, "/user/hand/right/input/skeleton/right" => Hand::Right, diff --git a/src/input/devices.rs b/src/input/devices.rs index 6ad8c007..dc8841c7 100644 --- a/src/input/devices.rs +++ b/src/input/devices.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::sync::Mutex; @@ -19,6 +20,8 @@ pub enum TrackedDeviceType { Hmd, Controller { hand: Hand, + hand_tracker: Option, + skeleton_cache: Mutex>>, }, #[cfg(feature = "monado")] GenericTracker { @@ -31,7 +34,7 @@ impl std::fmt::Debug for TrackedDeviceType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TrackedDeviceType::Hmd => write!(f, "HMD"), - TrackedDeviceType::Controller { hand } => write!(f, "Controller ({:?})", hand), + TrackedDeviceType::Controller { hand, .. } => write!(f, "Controller ({:?})", hand), #[cfg(feature = "monado")] TrackedDeviceType::GenericTracker { serial, .. } => { write!(f, "Generic Tracker ({})", serial.to_string_lossy()) @@ -158,8 +161,36 @@ impl TrackedDevice { *pose_cache } + pub fn get_hand_skeleton( + &self, + xr_data: &OpenXrData, + base: &xr::Space, + ) -> Option { + let TrackedDeviceType::Controller { + hand_tracker, + skeleton_cache, + .. + } = self.get_type() + else { + return None; + }; + let mut skeleton_cache = skeleton_cache.lock().unwrap(); + if let Some(skeleton) = skeleton_cache.get(&base.as_raw().into_raw()) { + return *skeleton; + } + + let joints = base + .locate_hand_joints(hand_tracker.as_ref()?, xr_data.display_time.get()) + .unwrap_or_default(); + skeleton_cache.insert(base.as_raw().into_raw(), joints); + joints + } + pub fn clear_pose_cache(&self) { std::mem::take(&mut *self.pose_cache.lock().unwrap()); + if let TrackedDeviceType::Controller { skeleton_cache, .. } = self.get_type() { + skeleton_cache.lock().unwrap().clear(); + } } pub fn has_connected_changed(&mut self) -> bool { @@ -184,7 +215,7 @@ impl TrackedDevice { fn get_string_property(&self, property: vr::ETrackedDeviceProperty) -> Option<&CStr> { let hand = match self.device_type { - TrackedDeviceType::Controller { hand } => hand, + TrackedDeviceType::Controller { hand, .. } => hand, _ => Hand::Left, }; diff --git a/src/input/profiles/knuckles.rs b/src/input/profiles/knuckles.rs index fafce4d5..9fa00830 100644 --- a/src/input/profiles/knuckles.rs +++ b/src/input/profiles/knuckles.rs @@ -144,6 +144,8 @@ impl InteractionProfile for Knuckles { .leftright("input/thumbstick/touch") .into_iter() .chain(stp.leftright("input/trackpad/touch")) + .chain(stp.leftright("input/a/touch")) + .chain(stp.leftright("input/b/touch")) .collect(), index_touch: stp.leftright("input/trigger/touch"), index_curl: stp.leftright("input/trigger/value"), diff --git a/src/input/skeletal.rs b/src/input/skeletal.rs index d74e1d5f..ffc1995e 100644 --- a/src/input/skeletal.rs +++ b/src/input/skeletal.rs @@ -14,19 +14,16 @@ use std::f32::consts::{FRAC_PI_2, PI}; use std::time::Instant; impl Input { - /// Returns false if hand tracking data couldn't be generated for some reason. pub(super) fn get_bones_from_hand_tracking( &self, session_data: &SessionData, space: vr::EVRSkeletalTransformSpace, - hand_tracker: &xr::HandTracker, hand: Hand, transforms: &mut [vr::VRBoneTransform_t], ) { use HandSkeletonBone::*; let pose_data = session_data.input_data.pose_data.get().unwrap(); - let display_time = self.openxr.display_time.get(); let devices = session_data.input_data.devices.read().unwrap(); let Some(controller) = devices.get_controller(hand) else { @@ -43,7 +40,7 @@ impl Input { return; }; - let Some(joints) = raw.locate_hand_joints(hand_tracker, display_time).unwrap() else { + let Some(joints) = controller.get_hand_skeleton(&self.openxr, &raw) else { self.get_estimated_bones(session_data, space, hand, transforms); return; }; @@ -165,6 +162,127 @@ impl Input { *self.skeletal_tracking_level.write().unwrap() = vr::EVRSkeletalTrackingLevel::Full; } + pub(super) fn get_bone_summary_from_hand_tracking( + &self, + session_data: &SessionData, + summary_type: vr::EVRSummaryType, + summary_data: &mut vr::VRSkeletalSummaryData_t, + hand: Hand, + ) { + let pose_data = session_data.input_data.pose_data.get().unwrap(); + let devices = session_data.input_data.devices.read().unwrap(); + + let Some(controller) = devices.get_controller(hand) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let Some(raw) = match hand { + Hand::Left => &pose_data.left_space, + Hand::Right => &pose_data.right_space, + } + .try_get_or_init_raw(&controller.interaction_profile, session_data, pose_data) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let Some(joints) = controller.get_hand_skeleton(&self.openxr, &raw) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let joints: Box<[_]> = joints + .into_iter() + .map(|joint_location| { + let position = joint_location.pose.position; + let orientation = joint_location.pose.orientation; + ( + Vec3::from_array([position.x, position.y, position.z]), + Quat::from_xyzw(orientation.x, orientation.y, orientation.z, orientation.w), + ) + }) + .collect(); + + // FIXME: calculate splay + summary_data.flFingerSplay.fill(0.2); + + for (i, out_curl) in summary_data.flFingerCurl.iter_mut().enumerate() { + // For Knuckles, the skeleton thumb tries to accurately match where the physical + // thumb is, e.g. the curl depends on which part of the touchpad is being touched, + // or how the thumbstick is being pushed, but in GetSkeletalSummaryData with + // EVRSummaryType::FromDevice the curl is set to 1 if any of touchpad, A/B, + // or thumbstick is being touched, so just use the skeletal input actions to + // determine the curl value for the thumb. + if i == 0 + && controller.interaction_profile.is_some_and(|p| { + p.profile_path() == "/interaction_profiles/valve/index_controller" + }) + { + *out_curl = self.get_finger_state(session_data, hand).thumb; + continue; + } + + let (metacarpal, proximal, tip) = match i { + 0 => ( + joints[xr::HandJoint::THUMB_METACARPAL], + joints[xr::HandJoint::THUMB_PROXIMAL], + joints[xr::HandJoint::THUMB_TIP], + ), + 1 => ( + joints[xr::HandJoint::INDEX_METACARPAL], + joints[xr::HandJoint::INDEX_PROXIMAL], + joints[xr::HandJoint::INDEX_TIP], + ), + 2 => ( + joints[xr::HandJoint::MIDDLE_METACARPAL], + joints[xr::HandJoint::MIDDLE_PROXIMAL], + joints[xr::HandJoint::MIDDLE_TIP], + ), + 3 => ( + joints[xr::HandJoint::RING_METACARPAL], + joints[xr::HandJoint::RING_PROXIMAL], + joints[xr::HandJoint::RING_TIP], + ), + 4 => ( + joints[xr::HandJoint::LITTLE_METACARPAL], + joints[xr::HandJoint::LITTLE_PROXIMAL], + joints[xr::HandJoint::LITTLE_TIP], + ), + _ => unreachable!(), + }; + + // Vector pointing from the knuckle to the metacarpal + let proximal_metacarpal_delta = metacarpal.0 - proximal.0; + // Vector pointing from the knuckle to the tip of the finger + let tip_proximal_delta = tip.0 - proximal.0; + + // The dotproduct will give us the angle between the two vectors + let dot = proximal_metacarpal_delta.dot(tip_proximal_delta); + let a = proximal_metacarpal_delta.length(); + let b = tip_proximal_delta.length(); + + let curl = if a == 0.0 || b == 0.0 { + // If the joints converge, say the finger is fully curled + 1.0 + } else { + // Isolate cos(angle) + let ang_cos = (dot / (a * b)).clamp(-1.0, 1.0); + // Convert the angle to radians + let ang = ang_cos.acos(); + + // When the finger is straight, the angle is 180deg (PI radians) + // When the finger is fully curled, the angle is (theoretically) 0 + 1.0 - (ang / PI) + }; + + // But in the real world, when a finger is curled, an angle of 0 is physically + // not possible. The curl maxes out at around 0.8, give or take some depending on + // the user's hands. Remap the value so >=0.8 (arbitrary value) means fully curled + const MAX_CURL: f32 = 0.8; + *out_curl = (curl / MAX_CURL).clamp(0.0, 1.0); + } + } + pub(super) fn get_estimated_bones( &self, session_data: &SessionData, @@ -212,6 +330,27 @@ impl Input { *self.skeletal_tracking_level.write().unwrap() = vr::EVRSkeletalTrackingLevel::Estimated; } + pub(super) fn get_estimated_bone_summary( + &self, + session_data: &SessionData, + _: vr::EVRSummaryType, + summary_data: &mut vr::VRSkeletalSummaryData_t, + hand: Hand, + ) { + let state = self.get_finger_state(session_data, hand); + + *summary_data = vr::VRSkeletalSummaryData_t { + flFingerSplay: [0.2; 4], + flFingerCurl: [ + state.thumb, + state.index, + state.middle, + state.ring, + state.pinky, + ], + }; + } + fn get_finger_state(&self, session_data: &SessionData, hand: Hand) -> FingerState { // Determines the speed at which fingers follow the input states // This value seems to feel right for both analog inputs and binary ones (like vive wands) diff --git a/src/openxr_data.rs b/src/openxr_data.rs index 333673de..ab498044 100644 --- a/src/openxr_data.rs +++ b/src/openxr_data.rs @@ -687,6 +687,15 @@ impl From for vr::ETrackedControllerRole { } } +impl From for xr::HandEXT { + fn from(hand: Hand) -> Self { + match hand { + Hand::Left => xr::HandEXT::LEFT, + Hand::Right => xr::HandEXT::RIGHT, + } + } +} + /// Taken from: https://github.com/bitshifter/glam-rs/issues/536 /// Decompose the rotation on to 2 parts. ///