/** * @file * * @author Mamadou Babaei * * @section LICENSE * * (The MIT License) * * Copyright (c) 2020 - 2024 SenseGlove * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * @section DESCRIPTION * * */ #include "SGTracking/SGXRTracker.h" #include "Containers/Array.h" #include "Containers/Map.h" #include "Engine/Engine.h" #include "Engine/SkeletalMesh.h" #include "Engine/World.h" #include "GameFramework/WorldSettings.h" #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 #include "IOpenXRHMD.h" #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */ #include "IOpenXRHMDModule.h" #include "IXRTrackingSystem.h" #include "OpenXRCore.h" #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 #include "OpenXRHMD.h" #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 */ #include "Runtime/Launch/Resources/Version.h" #include "SGCore/SGHapticGlove.h" #include "SGLog/SGLog.h" #include "SGSettings/SGSettings.h" #include "SGTracking/SGGloveTracker.h" #include "SGTracking/SGHMDTracker.h" #include "SGTracking/SGXRHandState.h" #include "SGTypes/SGCoreTypes.h" #include "SGTypes/SGTrackingTypes.h" #include "SGUtils/SGEngineUtils.h" #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 void EnumerateOpenXRApiLayers(TArray& OutProperties) { uint32 Count = 0; XR_ENSURE(xrEnumerateApiLayerProperties(0, &Count, nullptr)); OutProperties.SetNum(Count); for (auto& Prop: OutProperties) { Prop = XrApiLayerProperties{XR_TYPE_API_LAYER_PROPERTIES}; } XR_ENSURE(xrEnumerateApiLayerProperties(Count, &Count, OutProperties.GetData())); } #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 */ struct FSGXRTracker::FImpl { /************************ * Typedefs ************************/ // bool true == left, false == right typedef TPair FSGMotionSourceInfo; /************************ * Static methods ************************/ /** * Parses the enum name removing the prefix. */ static FName GetHandKeypointEnumName(const EHandKeypoint HandKeypoint); FORCEINLINE static FName GetHandKeypointEnumName(const int32 HandKeypointIndex) { return GetHandKeypointEnumName(static_cast(HandKeypointIndex)); } static FName GetControllerHandEnumName(const EControllerHand ControllerHand); FORCEINLINE static bool ShouldFallbackToHandTrackingIfNoGloveDetected() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = Settings->GetTrackingSettings().GloveTrackingSettings.bFallbackToHandTrackingIfNoGloveDetected; return bFallbackToHandTrackingIfNoGloveDetected; } FORCEINLINE static bool ShouldUseMoreSpecificMotionSourceNames() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bUseMoreSpecificMotionSourceNames = bFallbackToHandTrackingIfNoGloveDetected ? Settings->GetTrackingSettings().HandTrackingSettings.bUseMoreSpecificMotionSourceNames : false; return bUseMoreSpecificMotionSourceNames; } FORCEINLINE static bool ShouldSupportLegacyControllerMotionSources() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bSupportLegacyControllerMotionSources = bFallbackToHandTrackingIfNoGloveDetected ? Settings->GetTrackingSettings().HandTrackingSettings.bSupportLegacyControllerMotionSources : false; return bSupportLegacyControllerMotionSources; } FORCEINLINE static const FSGHandTrackingSettings& GetHandTrackingSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetTrackingSettings().HandTrackingSettings; } FORCEINLINE static const FSGVirtualHandSettings& GetVirtualHandSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetVirtualHandSettings(); } FORCEINLINE static const FSGWristTrackingSettings& GetWristTrackingSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetTrackingSettings().WristTrackingSettings; } FORCEINLINE static const FName& GetDefaultLeftHandTrackingMotionSourceName() { static const FName Name(TEXT("Left")); return Name; } FORCEINLINE static const FName& GetDefaultRightHandTrackingMotionSourceName() { static const FName Name(TEXT("Right")); return Name; } FORCEINLINE static const FName& GetDefaultHandTrackingMotionSourceName(const EControllerHand Hand) { const FName& Name{ Hand == EControllerHand::Right ? GetDefaultRightHandTrackingMotionSourceName() : GetDefaultLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetDefaultHandTrackingMotionSourceName(const bool bRight) { const FName& Name{ bRight ? GetDefaultRightHandTrackingMotionSourceName() : GetDefaultLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetLegacyControllerLeftHandTrackingMotionSourceName() { static const FName Name(TEXT("Left")); return Name; } FORCEINLINE static const FName& GetLegacyControllerRightHandTrackingMotionSourceName() { static const FName Name(TEXT("Right")); return Name; } FORCEINLINE static const FName& GetLegacyControllerHandTrackingMotionSourceName(const EControllerHand Hand) { const FName& Name{ Hand == EControllerHand::Right ? GetLegacyControllerRightHandTrackingMotionSourceName() : GetLegacyControllerLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetLegacyControllerHandTrackingMotionSourceName(const bool bRight) { const FName& Name{ bRight ? GetLegacyControllerRightHandTrackingMotionSourceName() : GetLegacyControllerLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetMoreSpecificLeftHandTrackingMotionSourceName() { static const FName Name(TEXT("HandTrackingLeft")); return Name; } FORCEINLINE static const FName& GetMoreSpecificRightHandTrackingMotionSourceName() { static const FName Name(TEXT("HandTrackingRight")); return Name; } FORCEINLINE static const FName& GetMoreSpecificHandTrackingMotionSourceName(const EControllerHand Hand) { const FName& Name{ Hand == EControllerHand::Right ? GetMoreSpecificRightHandTrackingMotionSourceName() : GetMoreSpecificLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetMoreSpecificHandTrackingMotionSourceName(const bool bRight) { const FName& Name{ bRight ? GetMoreSpecificRightHandTrackingMotionSourceName() : GetMoreSpecificLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetLeftHandTrackingMotionSourceName() { const bool bUseMoreSpecificMotionSourceNames = ShouldUseMoreSpecificMotionSourceNames(); const FName& Name{ bUseMoreSpecificMotionSourceNames ? GetMoreSpecificLeftHandTrackingMotionSourceName() : GetDefaultLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetRightHandTrackingMotionSourceName() { const bool bUseMoreSpecificMotionSourceNames = ShouldUseMoreSpecificMotionSourceNames(); const FName& Name{ bUseMoreSpecificMotionSourceNames ? GetMoreSpecificRightHandTrackingMotionSourceName() : GetDefaultRightHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static const FName& GetHandTrackingMotionSourceName(const EControllerHand Hand) { const FName& Name{ Hand == EControllerHand::Right ? GetRightHandTrackingMotionSourceName() : GetLeftHandTrackingMotionSourceName()}; return Name; } FORCEINLINE static const FName& GetHandTrackingMotionSourceName(const bool bRight) { const FName& Name{ bRight ? GetRightHandTrackingMotionSourceName() : GetLeftHandTrackingMotionSourceName() }; return Name; } FORCEINLINE static FName GetLeftPositionalTrackingHardwareMotionSourceName() { const FName Name{ GetControllerHandEnumName(GetWristTrackingSettings().LeftHandMotionSource) }; return Name; } FORCEINLINE static FName GetRightPositionalTrackingHardwareMotionSourceName() { const FName Name{ GetControllerHandEnumName(GetWristTrackingSettings().RightHandMotionSource) }; return Name; } FORCEINLINE static FName GetPositionalTrackingHardwareMotionSourceName(const EControllerHand Hand) { const FName Name{ Hand == EControllerHand::Right ? GetRightPositionalTrackingHardwareMotionSourceName() : GetLeftPositionalTrackingHardwareMotionSourceName()}; return Name; } FORCEINLINE static FName GetPositionalTrackingHardwareMotionSourceName(const bool bRight) { const FName& Name{ bRight ? GetRightPositionalTrackingHardwareMotionSourceName() : GetLeftPositionalTrackingHardwareMotionSourceName() }; return Name; } static ESGPositionalTrackingHardware GetPositionalTrackingHardware(); FORCEINLINE static USGHapticGlove* GetGlove(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetGlove(bRight) : nullptr; } FORCEINLINE static USGHapticGlove* GetGlove(const bool bRight) { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetGlove(bRight) : nullptr; } FORCEINLINE static USGHapticGlove* GetLeftGlove() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetLeftGlove() : nullptr; } FORCEINLINE static USGHapticGlove* GetRightGlove() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetRightGlove() : nullptr; } FORCEINLINE static bool IsGloveConnected(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsGloveConnected(bRight) : false; } FORCEINLINE static bool IsGloveConnected(const bool bRight) { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsGloveConnected(bRight) : false; } FORCEINLINE static bool IsLeftGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsLeftGloveConnected() : false; } FORCEINLINE static bool IsRightGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsRightGloveConnected() : false; } FORCEINLINE static bool IsAnyGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsAnyGloveConnected() : false; } static TArray GetFingerLengths( bool bRight, EHandKeypoint Joint2nd, EHandKeypoint Joint3rd, float DistalPhalanxLength); static TArray> GetFingerLengths(bool bRight); static TArray GetFingerStartTransforms(bool bRight); static TArray GetFingerStartPositions(bool bRight); static TArray GetFingerStartRotations(bool bRight); static USGBasicHandModel* GetHandModel(bool bRight); FORCEINLINE static USGBasicHandModel* GetHandModel(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; USGBasicHandModel* HandModel{GetHandModel(bRight)}; return HandModel; } static USGHandPose* GetHandPose(bool bRight); FORCEINLINE static USGHandPose* GetHandPose(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; USGHandPose* HandPose{GetHandPose(bRight)}; return HandPose; } FORCEINLINE static const FName& GetLeftMeshBoneName(const int32 JointIndex) { const FName& Name{GetVirtualHandSettings().MeshSettings.LeftHandBoneNames[JointIndex]}; return Name; } FORCEINLINE static const FName& GetLeftMeshBoneName(const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetLeftMeshBoneName(JointIndex)}; return Name; } FORCEINLINE static const FName& GetRightMeshBoneName(const int32 JointIndex) { const FName& Name{GetVirtualHandSettings().MeshSettings.RightHandBoneNames[JointIndex]}; return Name; } FORCEINLINE static const FName& GetRightMeshBoneName(const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetRightMeshBoneName(JointIndex)}; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const bool bRight, const int32 Joint) { const FName& Name{ bRight ? GetRightMeshBoneName(Joint) : GetLeftMeshBoneName(Joint) }; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const bool bRight, const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetMeshBoneName(bRight, JointIndex)}; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FName& Name{GetMeshBoneName(bRight, JointIndex)}; return Name; } static FTransform GetMeshRawBoneRefTransform(bool bRight, const FName& BoneName); FORCEINLINE static FTransform GetMeshRawBoneRefTransform(const bool bRight, const int32 JointIndex) { const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform& Transform{GetMeshRawBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshRawBoneRefTransform(const bool bRight, const EHandKeypoint Keypoint) { const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform& Transform{GetMeshRawBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshRawBoneRefTransform(const EControllerHand Hand, const FName& BoneName) { const bool bRight = Hand == EControllerHand::Right; const FTransform& Transform{GetMeshRawBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshRawBoneRefTransform(const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform& Transform{GetMeshRawBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshRawBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Keypoint) { const bool bRight = Hand == EControllerHand::Right; const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform& Transform{GetMeshRawBoneRefTransform(bRight, BoneName)}; return Transform; } static FTransform GetMeshWristChildBoneRefTransform(bool bRight, const FName& BoneName); FORCEINLINE static FTransform GetMeshWristChildBoneRefTransform( const bool bRight, const int32 JointIndex) { const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform Transform{GetMeshWristChildBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshWristChildBoneRefTransform( const bool bRight, const EHandKeypoint Keypoint) { const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform Transform{GetMeshWristChildBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshWristChildBoneRefTransform( const EControllerHand Hand, const FName& BoneName) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshWristChildBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshWristChildBoneRefTransform( const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshWristChildBoneRefTransform(bRight, JointIndex)}; return Transform; } FORCEINLINE static FTransform GetMeshWristChildBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Keypoint) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshWristChildBoneRefTransform(bRight, Keypoint)}; return Transform; } static FTransform GetMeshFingerStartBoneRefTransform( bool bRight, EHandKeypoint Parent, const FName& BoneName); FORCEINLINE static FTransform GetMeshFingerStartBoneRefTransform( const bool bRight, const EHandKeypoint Parent, const int32 JointIndex) { const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform Transform{GetMeshFingerStartBoneRefTransform(bRight, Parent, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshFingerStartBoneRefTransform( const bool bRight, const EHandKeypoint Parent, const EHandKeypoint Keypoint) { const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform Transform{GetMeshFingerStartBoneRefTransform(bRight, Parent, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshFingerStartBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Parent, const FName& BoneName) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshFingerStartBoneRefTransform(bRight, Parent, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshFingerStartBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Parent, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshFingerStartBoneRefTransform(bRight, Parent, JointIndex)}; return Transform; } FORCEINLINE static FTransform GetMeshFingerStartBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Parent, const EHandKeypoint Keypoint) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshFingerStartBoneRefTransform(bRight, Parent, Keypoint)}; return Transform; } /************************ * Member variables ************************/ bool bHandTrackingAvailable; PFN_xrCreateHandTrackerEXT XRCreateHandTrackerEXT; PFN_xrDestroyHandTrackerEXT XRDestroyHandTrackerEXT; PFN_xrLocateHandJointsEXT XRLocateHandJointsEXT; IXRTrackingSystem* XRTrackingSystem; TSharedPtr MessageHandler; int32 DeviceIndex; int32 CurrentHandTrackingDataIndex; TArray BoneParents; TArray BoneKeypoints; TMap MotionSourceToKeypointMap; FSGXRHandState HandStates[2]; TArray LeftAnimationTransforms; TArray RightAnimationTransforms; /************************ * Owner object ************************/ FSGXRTracker* Owner; /************************ * Constructor / Destructor ************************/ explicit FImpl(FSGXRTracker* InOwner); ~FImpl(); /************************ * Default copy constructor & copy assignment operator ************************/ FImpl(const FImpl& Rhs) = delete; FImpl& operator=(const FImpl& Rhs) = delete; /************************ * Methods ************************/ FORCEINLINE const FSGXRHandState& GetLeftHandState() const { return HandStates[0]; } FORCEINLINE FSGXRHandState& GetLeftHandState() { return HandStates[0]; } FORCEINLINE const FSGXRHandState& GetRightHandState() const { return HandStates[1]; } FORCEINLINE FSGXRHandState& GetRightHandState() { return HandStates[1]; } void BuildMotionSourceToKeypointMap(); bool CanOutputTrackingData(EControllerHand Hand) const; bool UpdateXRHandTrackingData(FSGXRHandState& OutHandState, const XrHandJointsLocateInfoEXT& LocateInfo) const; bool GetControllerTransform(const FName MotionSource, const float WorldToMetersScale, FTransform& OutTransform, bool& bOutMotionSourceHandTracking) const; bool GetWristTransform(bool bRight, FTransform& OutTransform) const; bool UpdateSGJointData(FSGXRHandState& OutHandState, const EHandKeypoint Joint, const FTransform& JointTransform, const FTransform& WristTransform) const; bool UpdateSGWristJointData(FSGXRHandState& OutHandState, const FTransform& WristTransform) const; bool UpdateSGPalmJointData(FSGXRHandState& OutHandState, bool bRight, const FTransform& WristTransform) const; bool UpdateSGThumbJointData( FSGXRHandState& OutHandState, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const; bool UpdateSGIndexJointData( FSGXRHandState& OutHandState, bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const; bool UpdateSGMiddleJointData( FSGXRHandState& OutHandState, bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const; bool UpdateSGRingJointData( FSGXRHandState& OutHandState, bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const; bool UpdateSGLittleJointData( FSGXRHandState& OutHandState, bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const; bool UpdateSGHandState(bool bRight, FSGXRHandState& OutHandState) const; }; FName FSGXRTracker::FImpl::GetHandKeypointEnumName(const EHandKeypoint HandKeypoint) { static UEnum* EnumPtr{StaticEnum()}; ensureAlwaysMsgf(EnumPtr, TEXT("Failed to find the EHandKeypoint enum!")); static const int32 EnumNameLength = FString{TEXT("EHandKeypoint::")}.Len(); const FName EnumItemName{ EnumPtr->GetNameByValue(static_cast(HandKeypoint)) }; const FString EnumItemString{EnumItemName.ToString()}; const int32 EnumItemStringLength = EnumItemString.Len(); const int32 EnumItemNameLength = EnumItemStringLength - EnumNameLength; const FName ParsedName{*EnumItemString.Right(EnumItemNameLength)}; return ParsedName; } FName FSGXRTracker::FImpl::GetControllerHandEnumName(const EControllerHand ControllerHand) { static UEnum* EnumPtr{StaticEnum()}; ensureAlwaysMsgf(EnumPtr, TEXT("Failed to find the EControllerHand enum!")); static const int32 EnumNameLength = FString{TEXT("EControllerHand::")}.Len(); const FName EnumItemName{ EnumPtr->GetNameByValue(static_cast(ControllerHand)) }; const FString EnumItemString{EnumItemName.ToString()}; const int32 EnumItemStringLength = EnumItemString.Len(); const int32 EnumItemNameLength = EnumItemStringLength - EnumNameLength; const FName ParsedName{*EnumItemString.Right(EnumItemNameLength)}; return ParsedName; } ESGPositionalTrackingHardware FSGXRTracker::FImpl::GetPositionalTrackingHardware() { ESGPositionalTrackingHardware TrackerHardware = GetWristTrackingSettings().TrackingHardware; if (TrackerHardware == ESGPositionalTrackingHardware::None) { const ESGHeadMountedDisplayDevice HmdDevice = FSGHMDTracker::GetDeviceType(); switch (HmdDevice) { case ESGHeadMountedDisplayDevice::MetaQuest2: TrackerHardware = ESGPositionalTrackingHardware::Quest2Controller; break; case ESGHeadMountedDisplayDevice::MetaQuestPro: TrackerHardware = ESGPositionalTrackingHardware::QuestProController; break; case ESGHeadMountedDisplayDevice::MetaQuest3: TrackerHardware = ESGPositionalTrackingHardware::Quest3Controller; break; case ESGHeadMountedDisplayDevice::HtcVivePro: TrackerHardware = ESGPositionalTrackingHardware::ViveTracker; break; case ESGHeadMountedDisplayDevice::HtcViveFocus3: // Fall through case ESGHeadMountedDisplayDevice::HtcViveXRElite: TrackerHardware = ESGPositionalTrackingHardware::ViveFocus3WristTracker; break; default: TrackerHardware = ESGPositionalTrackingHardware::Custom; break; } } return TrackerHardware; } TArray FSGXRTracker::FImpl::GetFingerLengths( const bool bRight, const EHandKeypoint Joint2nd, const EHandKeypoint Joint3rd, const float DistalPhalanxLength) { const FTransform& Joint2ndTransform{GetMeshRawBoneRefTransform(bRight, Joint2nd)}; const FVector& Joint2ndLocation{Joint2ndTransform.GetLocation()}; const FTransform& Joint3rdTransform{GetMeshRawBoneRefTransform(bRight, Joint3rd)}; const FVector& Joint3rdLocation{Joint3rdTransform.GetLocation()}; const float PhalanxLength1st = static_cast(Joint2ndLocation.Size()); const float PhalanxLength2nd = static_cast(Joint3rdLocation.Size()); const TArray Lengths{PhalanxLength1st, PhalanxLength2nd, DistalPhalanxLength}; return Lengths; } TArray> FSGXRTracker::FImpl::GetFingerLengths(const bool bRight) { const FSGVirtualHandSettings& Settings{GetVirtualHandSettings()}; const TArray> Lengths{ GetFingerLengths(bRight, EHandKeypoint::ThumbProximal, EHandKeypoint::ThumbDistal, Settings.MeshSettings.DistalPhalangesLengthSettings.Thumb ), GetFingerLengths(bRight, EHandKeypoint::IndexIntermediate, EHandKeypoint::IndexDistal, Settings.MeshSettings.DistalPhalangesLengthSettings.Index ), GetFingerLengths(bRight, EHandKeypoint::MiddleIntermediate, EHandKeypoint::MiddleDistal, Settings.MeshSettings.DistalPhalangesLengthSettings.Middle ), GetFingerLengths(bRight, EHandKeypoint::RingIntermediate, EHandKeypoint::RingDistal, Settings.MeshSettings.DistalPhalangesLengthSettings.Ring ), GetFingerLengths(bRight, EHandKeypoint::LittleIntermediate, EHandKeypoint::LittleDistal, Settings.MeshSettings.DistalPhalangesLengthSettings.Little ), }; return Lengths; } TArray FSGXRTracker::FImpl::GetFingerStartTransforms(const bool bRight) { FTransform ThumbStartTransform{ GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::ThumbMetacarpal).GetLocation() }; FTransform IndexStartTransform{ GetMeshFingerStartBoneRefTransform(bRight, EHandKeypoint::IndexMetacarpal, EHandKeypoint::IndexProximal) }; FTransform MiddleStartTransform{ GetMeshFingerStartBoneRefTransform(bRight, EHandKeypoint::MiddleMetacarpal, EHandKeypoint::MiddleProximal) }; FTransform RingStartTransform{ GetMeshFingerStartBoneRefTransform(bRight, EHandKeypoint::RingMetacarpal, EHandKeypoint::RingProximal) }; FTransform LittleStartTransform{ GetMeshFingerStartBoneRefTransform(bRight, EHandKeypoint::LittleMetacarpal, EHandKeypoint::LittleProximal) }; TArray Transforms{ MoveTemp(ThumbStartTransform), MoveTemp(IndexStartTransform), MoveTemp(MiddleStartTransform), MoveTemp(RingStartTransform), MoveTemp(LittleStartTransform), }; return Transforms; } TArray FSGXRTracker::FImpl::GetFingerStartPositions(const bool bRight) { const TArray StartTransforms{GetFingerStartTransforms(bRight)}; ensureAlwaysMsgf(StartTransforms.Num() == 5, TEXT("Invalid number of finger start transforms!")); const TArray StartPositions{ StartTransforms[0].GetLocation(), StartTransforms[1].GetLocation(), StartTransforms[2].GetLocation(), StartTransforms[3].GetLocation(), StartTransforms[4].GetLocation(), }; return StartPositions; } TArray FSGXRTracker::FImpl::GetFingerStartRotations(const bool bRight) { const TArray StartRotations{ FQuat::Identity, FQuat::Identity, FQuat::Identity, FQuat::Identity, FQuat::Identity, }; return StartRotations; } USGBasicHandModel* FSGXRTracker::FImpl::GetHandModel(const bool bRight) { const TArray> FingerLengths{GetFingerLengths(bRight)}; const TArray StartPositions{GetFingerStartPositions(bRight)}; const TArray StartRotations{GetFingerStartRotations(bRight)}; UWorld* World{FSGEngineUtils::GetWorld()}; USGBasicHandModel* HandModel{ USGBasicHandModel::NewBasicHandModel(World, bRight, FingerLengths, StartPositions, StartRotations) }; return HandModel; } USGHandPose* FSGXRTracker::FImpl::GetHandPose(const bool bRight) { const USGHapticGlove* Glove{GetGlove(bRight)}; if (!IsValid(Glove)) { return nullptr; } UWorld* World{FSGEngineUtils::GetWorld()}; const USGBasicHandModel* HandModel{GetHandModel(bRight)}; USGHandPose* HandPose{nullptr}; const bool bGotHandPose = Glove->GetHandPose(World, HandModel, HandPose); if (!bGotHandPose || !IsValid(HandPose)) { return nullptr; } return HandPose; } FTransform FSGXRTracker::FImpl::GetMeshRawBoneRefTransform(const bool bRight, const FName& BoneName) { const FSGVirtualHandMeshSettings& MeshSettings{GetVirtualHandSettings().MeshSettings}; const TSoftObjectPtr& HandMesh{ bRight ? MeshSettings.LeftHandReferenceMesh : MeshSettings.RightHandReferenceMesh }; if (HandMesh.IsValid()) { const FReferenceSkeleton& ReferenceSkeleton{HandMesh->GetRefSkeleton()}; const int32 BoneIndex = ReferenceSkeleton.FindBoneIndex(BoneName); if (BoneIndex == INDEX_NONE) { SGLOG_WARNING(FString::Printf(TEXT("Could not find a bone index for the '%s' joint!"), *BoneName.ToString())); } const TArray Transforms{ReferenceSkeleton.GetRefBonePose()}; ensureAlwaysMsgf(Transforms.Num() > 0 && Transforms.Num() > BoneIndex, TEXT("Invalid bone index '%d' or no transforms found!"), BoneIndex); const FTransform& Transform{Transforms[BoneIndex]}; return Transform; } ensureAlwaysMsgf( bRight ? MeshSettings.RightHandDefaultReferenceBoneTransforms.Contains(BoneName) : MeshSettings.LeftHandDefaultReferenceBoneTransforms.Contains(BoneName), TEXT("Invalid bone name '%s'!"), *BoneName.ToString()); const FTransform& Transform{ bRight ? MeshSettings.RightHandDefaultReferenceBoneTransforms[BoneName] : MeshSettings.LeftHandDefaultReferenceBoneTransforms[BoneName] }; return Transform; } FTransform FSGXRTracker::FImpl::GetMeshWristChildBoneRefTransform(const bool bRight, const FName& BoneName) { const FSGVirtualHandMeshSettings& MeshSettings{GetVirtualHandSettings().MeshSettings}; FQuat RootRotationCorrection{MeshSettings.RootBoneRotationCorrection}; const FTransform& RootBoneRefTransform{GetMeshRawBoneRefTransform(bRight, EHandKeypoint::Wrist)}; const FTransform& RawTransform{GetMeshRawBoneRefTransform(bRight, BoneName)}; FQuat CorrectedRootBoneRefRotation{MoveTemp(RootRotationCorrection) * RootBoneRefTransform.GetRotation()}; FVector CorrectedLocation{MoveTemp(CorrectedRootBoneRefRotation) * RawTransform.GetLocation()}; const FTransform Transform{RawTransform.GetRotation(), MoveTemp(CorrectedLocation)}; return Transform; } FTransform FSGXRTracker::FImpl::GetMeshFingerStartBoneRefTransform( const bool bRight, const EHandKeypoint Parent, const FName& BoneName) { const FSGVirtualHandMeshSettings& MeshSettings{GetVirtualHandSettings().MeshSettings}; FQuat RootRotationCorrection{MeshSettings.RootBoneRotationCorrection}; const FTransform& RootBoneRefTransform{GetMeshRawBoneRefTransform(bRight, EHandKeypoint::Wrist)}; FQuat CorrectedRootBoneRefRotation{MoveTemp(RootRotationCorrection) * RootBoneRefTransform.GetRotation()}; FTransform CorrectedRootBoneRefTransform{ MoveTemp(CorrectedRootBoneRefRotation), RootBoneRefTransform.GetLocation() }; FTransform ParentBoneRefTransform{ GetMeshRawBoneRefTransform(bRight, Parent) * MoveTemp(CorrectedRootBoneRefTransform) }; const FTransform Transform{GetMeshRawBoneRefTransform(bRight, BoneName) * MoveTemp(ParentBoneRefTransform)}; return Transform; } bool FSGXRTracker::GetMotionControllerData( UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData) { OutMotionControllerData.DeviceName = NAME_None; OutMotionControllerData.ApplicationInstanceID = FApp::GetInstanceId(); OutMotionControllerData.DeviceVisualType = EXRVisualType::Controller; OutMotionControllerData.TrackingStatus = ETrackingStatus::NotTracked; OutMotionControllerData.HandIndex = Hand; OutMotionControllerData.bValid = false; if (!IsValid(WorldContextObject)) { return false; } IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr}; if (!XRTrackingSystem) { return false; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 IOpenXRHMD* OpenXRHMD{XRTrackingSystem->GetIOpenXRHMD()}; #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */ FOpenXRHMD* OpenXRHMD{static_cast(GEngine->XRSystem.Get())}; #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */ if (!OpenXRHMD) { return false; } TArray Devices; const int32 HandIndex = static_cast(Hand); const bool bGotDevices = XRTrackingSystem->EnumerateTrackedDevices(Devices, EXRTrackedDeviceType::Controller); if (bGotDevices && Devices.IsValidIndex(HandIndex)) { const XrSession Session{OpenXRHMD->GetSession()}; if (Session) { XrInteractionProfileState Profile = {XR_TYPE_INTERACTION_PROFILE_STATE}; const XrPath TopLevelUserPath{OpenXRHMD->GetTrackedDevicePath(Devices[HandIndex])}; const XrResult CurrentInteractionProfile{ xrGetCurrentInteractionProfile(Session, TopLevelUserPath, &Profile) }; if (XR_SUCCEEDED(CurrentInteractionProfile) && Profile.interactionProfile != XR_NULL_PATH) { OutMotionControllerData.DeviceName = FOpenXRPath(Profile.interactionProfile); } } } static const FName HandTrackerName{"OpenXRHandTracking"}; TArray HandTrackers = IModularFeatures::Get().GetModularFeatureImplementations( IHandTracker::GetModularFeatureName()); IHandTracker* HandTracker{nullptr}; for (IHandTracker* HandTrackerIterator: HandTrackers) { if (!HandTrackerIterator) { continue; } const FName DeviceTypeName{HandTrackerIterator->GetHandTrackerDeviceTypeName()}; if (DeviceTypeName != HandTrackerName) { continue; } FSGXRTracker* SGXRTracker{static_cast(HandTrackerIterator)}; if (!SGXRTracker) { continue; } HandTracker = SGXRTracker; break; } if (Hand == EControllerHand::Left || Hand == EControllerHand::Right) { static const FName MotionControllerName("OpenXR"); TArray MotionControllers{ IModularFeatures::Get().GetModularFeatureImplementations( IMotionController::GetModularFeatureName()) }; IMotionController* MotionController{nullptr}; for (IMotionController* MotionControllerIterator: MotionControllers) { if (!MotionControllerIterator) { continue; } const FName DeviceTypeName{MotionControllerIterator->GetMotionControllerDeviceTypeName()}; if (DeviceTypeName != MotionControllerName) { continue; } MotionController = MotionControllerIterator; break; } const float WorldToMeters = XRTrackingSystem->GetWorldToMetersScale(); if (MotionController) { const FTransform TrackingToWorld{XRTrackingSystem->GetTrackingToWorldTransform()}; { const FName AimSource{Hand == EControllerHand::Left ? FName{TEXT("LeftAim")} : FName{TEXT("RightAim")}}; FRotator AimSourceRotation; FVector AimSourcePosition; const bool bGotAimOrientationAndPosition = MotionController->GetControllerOrientationAndPosition( 0, AimSource, AimSourceRotation, AimSourcePosition, WorldToMeters); if (bGotAimOrientationAndPosition) { OutMotionControllerData.AimPosition = TrackingToWorld.TransformPosition(AimSourcePosition); OutMotionControllerData.AimRotation = TrackingToWorld.TransformRotation(AimSourceRotation.Quaternion()); } OutMotionControllerData.bValid |= bGotAimOrientationAndPosition; } const FName GripSource{Hand == EControllerHand::Left ? FName{TEXT("LeftGrip")} : FName{TEXT("RightGrip")}}; { FRotator GripRotation; FVector GripPosition; const bool bGotGripSourceOrientationAndPosition = MotionController->GetControllerOrientationAndPosition( 0, GripSource, GripRotation, GripPosition, WorldToMeters); if (bGotGripSourceOrientationAndPosition) { OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(GripPosition); OutMotionControllerData.GripRotation = TrackingToWorld.TransformRotation(GripRotation.Quaternion()); } OutMotionControllerData.bValid |= bGotGripSourceOrientationAndPosition; } { const FName PalmSource{ Hand == EControllerHand::Left ? FName{TEXT("LeftPalm")} : FName{TEXT("RightPalm")} }; FRotator PalmRotation; FVector PalmPosition; const bool bGotPalmOrientationAndPosition = MotionController->GetControllerOrientationAndPosition( 0, PalmSource, PalmRotation, PalmPosition, WorldToMeters); if (bGotPalmOrientationAndPosition) { OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(PalmPosition); OutMotionControllerData.GripRotation = TrackingToWorld.TransformRotation(PalmRotation.Quaternion()); } OutMotionControllerData.bValid |= bGotPalmOrientationAndPosition; } OutMotionControllerData.TrackingStatus = MotionController->GetControllerTrackingStatus(0, GripSource); } const bool bHandTrackingStateValid = HandTracker && HandTracker->IsHandTrackingStateValid(); if (bHandTrackingStateValid) { OutMotionControllerData.DeviceVisualType = EXRVisualType::Hand; OutMotionControllerData.bValid = HandTracker->GetAllKeypointStates( Hand, OutMotionControllerData.HandKeyPositions, OutMotionControllerData.HandKeyRotations, OutMotionControllerData.HandKeyRadii); if (!ensureAlwaysMsgf( !OutMotionControllerData.bValid || (OutMotionControllerData.HandKeyPositions.Num() == EHandKeypointCount && OutMotionControllerData.HandKeyRotations.Num() == EHandKeypointCount && OutMotionControllerData.HandKeyRadii.Num() == EHandKeypointCount), TEXT("Corrupted OutMotionControllerData!"))) { OutMotionControllerData.bValid = false; return false; } } } if (OutMotionControllerData.bValid) { OutMotionControllerData.TrackingStatus = ETrackingStatus::Tracked; } /// TODO // This is reportedly a WMR-specific convenience function for rapid prototyping. Not sure if it's useful for OpenXR. OutMotionControllerData.bIsGrasped = false; return OutMotionControllerData.bValid; } FSGXRTracker::FSGXRTracker(const TSharedRef& InMessageHandler) : Pimpl(TUniquePtr(new FImpl{this}, PimplDeleter)) { Pimpl->MessageHandler = InMessageHandler; // Register the modular features manually. IModularFeatures::Get().RegisterModularFeature(IHandTracker::GetModularFeatureName(), static_cast(this)); IModularFeatures::Get().RegisterModularFeature(IMotionController::GetModularFeatureName(), static_cast(this)); IModularFeatures::Get().RegisterModularFeature(IOpenXRExtensionPlugin::GetModularFeatureName(), static_cast(this)); // We're implicitly requiring that the OpenXRPlugin has been loaded and initialized at this point. if (!IOpenXRHMDModule::Get().IsAvailable()) { SGLOG_FATAL("The OpenXRHMD plugin isn't available!"); ensureAlwaysMsgf(false, TEXT("The OpenXRHMD plugin isn't available!")); } } FSGXRTracker::FSGXRTracker(FSGXRTracker&& Rhs) SG_NOEXCEPT = default; FSGXRTracker::~FSGXRTracker() = default; FSGXRTracker& FSGXRTracker::operator=(FSGXRTracker&& Rhs) SG_NOEXCEPT = default; bool FSGXRTracker::IsHandTrackingSupportedByDevice() const { const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bAnyGloveConnected = FImpl::IsAnyGloveConnected(); const bool bHandTrackingAvailable = Pimpl->bHandTrackingAvailable; const bool bHandTrackingSupportedByDevice = bAnyGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bHandTrackingAvailable); return bHandTrackingSupportedByDevice; } const FSGXRHandState& FSGXRTracker::GetLeftHandState() const { return Pimpl->GetLeftHandState(); } const FSGXRHandState& FSGXRTracker::GetRightHandState() const { return Pimpl->GetRightHandState(); } FString FSGXRTracker::GetDisplayName() { static const FString Name{TEXT("SenseGloveTracking")}; return Name; } bool FSGXRTracker::GetRequiredExtensions(TArray& OutExtensions) { TArray Properties; EnumerateOpenXRApiLayers(Properties); // Some API layers can crash the loader when enabled, if they're present we shouldn't enable the extension. for (const XrApiLayerProperties& Layer: Properties) { if (FCStringAnsi::Strstr(Layer.layerName, "XR_APILAYER_VIVE_hand_tracking") && Layer.layerVersion <= 1) { return false; } } OutExtensions.Add("XR_EXT_hand_tracking"); return true; } const void* FSGXRTracker::OnGetSystem(const XrInstance InInstance, const void* InNext) { // Store the extension open xr calls to member function pointers for convenient use. XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrCreateHandTrackerEXT", (PFN_xrVoidFunction*)&Pimpl->XRCreateHandTrackerEXT)); XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrDestroyHandTrackerEXT", (PFN_xrVoidFunction*)&Pimpl->XRDestroyHandTrackerEXT)); XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrLocateHandJointsEXT", (PFN_xrVoidFunction*)&Pimpl->XRLocateHandJointsEXT)); return InNext; } const void* FSGXRTracker::OnCreateSession(const XrInstance InInstance, const XrSystemId InSystem, const void* InNext) { // Need to wait until the EHandKeypoint enum has been loaded, so we do this here rather than in the constructor // which runs too early. Pimpl->BuildMotionSourceToKeypointMap(); XrSystemHandTrackingPropertiesEXT HandTrackingSystemProperties{ XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT}; XrSystemProperties systemProperties{XR_TYPE_SYSTEM_PROPERTIES, &HandTrackingSystemProperties}; XR_ENSURE(xrGetSystemProperties(InInstance, InSystem, &systemProperties)); Pimpl->bHandTrackingAvailable = HandTrackingSystemProperties.supportsHandTracking == XR_TRUE; return InNext; } const void* FSGXRTracker::OnBeginSession(const XrSession InSession, const void* InNext) { const bool bHandTrackingAvailable = Pimpl->bHandTrackingAvailable; if (bHandTrackingAvailable) { static FName SystemName(TEXT("OpenXR")); if (IsValid(GEngine) && GEngine->XRSystem.IsValid() && (GEngine->XRSystem->GetSystemName() == SystemName)) { Pimpl->XRTrackingSystem = GEngine->XRSystem.Get(); } // Create a hand tracker for the left hand that tracks default set of hand joints. FSGXRHandState& LeftHandState{Pimpl->GetLeftHandState()}; { XrHandTrackerCreateInfoEXT CreateInfo{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT}; CreateInfo.hand = XR_HAND_LEFT_EXT; CreateInfo.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT; XR_ENSURE(Pimpl->XRCreateHandTrackerEXT(InSession, &CreateInfo, &LeftHandState.HandTracker)); } // Create a hand tracker for the right hand that tracks default set of hand joints. FSGXRHandState& RightHandState{Pimpl->GetRightHandState()}; { XrHandTrackerCreateInfoEXT CreateInfo{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT}; CreateInfo.hand = XR_HAND_RIGHT_EXT; CreateInfo.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT; XR_ENSURE(Pimpl->XRCreateHandTrackerEXT(InSession, &CreateInfo, &RightHandState.HandTracker)); } } return InNext; } void FSGXRTracker::UpdateDeviceLocations( const XrSession InSession, const XrTime DisplayTime, const XrSpace TrackingSpace) { if (!IsHandTrackingStateValid()) { return; } XrHandJointsLocateInfoEXT LocateInfo{XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT}; LocateInfo.baseSpace = TrackingSpace; LocateInfo.time = DisplayTime; const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bLeftGloveConnected = FImpl::IsLeftGloveConnected(); if (bLeftGloveConnected) { /// NOTE /// It's necessary to call the UpdateXRHandTrackingData first for the wrist tracking to work properly when the /// motion controllers are not active and we rely on the OpenXRHandTracking data for wrist tracking. /// See the NOTE inside the FSGXRTracker::FImpl::GetWristTransform method. Pimpl->UpdateXRHandTrackingData(Pimpl->GetLeftHandState(), LocateInfo); Pimpl->UpdateSGHandState(false, Pimpl->GetLeftHandState()); } else { if (bFallbackToHandTrackingIfNoGloveDetected) { Pimpl->UpdateXRHandTrackingData(Pimpl->GetLeftHandState(), LocateInfo); } } const bool bRightGloveConnected = FImpl::IsRightGloveConnected(); if (bRightGloveConnected) { /// NOTE /// It's necessary to call the UpdateXRHandTrackingData first for the wrist tracking to work properly when the /// motion controllers are not active and we rely on the OpenXRHandTracking data for wrist tracking. /// See the NOTE inside the FSGXRTracker::FImpl::GetWristLocationAndRotation method. Pimpl->UpdateXRHandTrackingData(Pimpl->GetRightHandState(), LocateInfo); Pimpl->UpdateSGHandState(true, Pimpl->GetRightHandState()); } else { if (bFallbackToHandTrackingIfNoGloveDetected) { Pimpl->UpdateXRHandTrackingData(Pimpl->GetRightHandState(), LocateInfo); } } } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 bool FSGXRTracker::GetControllerOrientationAndPosition(const int32 ControllerIndex, const FName MotionSource, FRotator& OutOrientation, FVector& OutPosition, const float WorldToMetersScale) const #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ bool FSGXRTracker::GetControllerOrientationAndPosition(const int32 ControllerIndex, const EControllerHand DeviceHand, FRotator& OutOrientation, FVector& OutPosition, const float WorldToMetersScale) const #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ { if (!IsHandTrackingStateValid()) { return false; } // Hand tracking currently does not support late update. Current hand tracking systems have latency that make it // pointless. if (!IsInGameThread()) { return false; } if (ControllerIndex != Pimpl->DeviceIndex) { return false; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 const FName& MotionSource{FImpl::GetEControllerHandEnumName(DeviceHand)}; #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 */ FTransform ControllerTransform; bool bMotionSourceHandTracking; const bool bGotTransform = Pimpl->GetControllerTransform(MotionSource, WorldToMetersScale, ControllerTransform, bMotionSourceHandTracking); (void) bMotionSourceHandTracking; OutOrientation = ControllerTransform.GetRotation().Rotator(); OutPosition = ControllerTransform.GetLocation(); return bGotTransform; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 ETrackingStatus FSGXRTracker::GetControllerTrackingStatus(const int32 ControllerIndex, const FName MotionSource) const #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ ETrackingStatus FSGXRTracker::GetControllerTrackingStatus(const int32 ControllerIndex, const EControllerHand DeviceHand) const #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ { if (!IsHandTrackingStateValid()) { return ETrackingStatus::NotTracked; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 const FName& MotionSource{FImpl::GetEControllerHandEnumName(DeviceHand)}; #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 */ const FImpl::FSGMotionSourceInfo* KeyPointInfoPtr{Pimpl->MotionSourceToKeypointMap.Find(MotionSource)}; if (!KeyPointInfoPtr) { return ETrackingStatus::NotTracked; } const bool bRight = !KeyPointInfoPtr->Value; const bool bGlovePresent = (bRight && FImpl::IsRightGloveConnected()) || (!bRight && FImpl::IsLeftGloveConnected()); if (bGlovePresent) { return ETrackingStatus::Tracked; } const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); if (bFallbackToHandTrackingIfNoGloveDetected) { const FSGXRHandState& HandState{bRight ? GetRightHandState() : GetLeftHandState()}; return HandState.bReceivedJointPoses ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked; } return ETrackingStatus::NotTracked; } FName FSGXRTracker::GetMotionControllerDeviceTypeName() const { const static FName DefaultName(TEXT("OpenXRHandTracking")); return DefaultName; } void FSGXRTracker::EnumerateSources(TArray& SourcesOut) const { ensureAlwaysMsgf(IsInGameThread(), TEXT("Is not in the game thread!")); SourcesOut.Reserve(SourcesOut.Num() + (EHandKeypointCount * 2)); const FName& Left(FImpl::GetLeftHandTrackingMotionSourceName()); const FName& Right(FImpl::GetRightHandTrackingMotionSourceName()); for (int32 Keypoint = 0; Keypoint < EHandKeypointCount; Keypoint++) { const FName EnumName{FImpl::GetHandKeypointEnumName(Keypoint)}; const FString EnumString{EnumName.ToString()}; const FString StringLeft{FString::Printf(TEXT("%s%s"), *Left.ToString(), *EnumString)}; const FString StringRight{FString::Printf(TEXT("%s%s"), *Right.ToString(), *EnumString)}; FName SourceL(*(StringLeft)); FName SourceR(*(StringRight)); SourcesOut.Add(MoveTemp(SourceL)); SourcesOut.Add(MoveTemp(SourceR)); } } void FSGXRTracker::Tick(const float DeltaTime) { } void FSGXRTracker::SendControllerEvents() { } void FSGXRTracker::SetMessageHandler(const TSharedRef& InMessageHandler) { Pimpl->MessageHandler = InMessageHandler; } bool FSGXRTracker::Exec(UWorld* InWorld, const TCHAR* Cmd, FOutputDevice& Ar) { return false; } void FSGXRTracker::SetChannelValue( const int32 ControllerId, const FForceFeedbackChannelType ChannelType, const float Value) { } void FSGXRTracker::SetChannelValues(const int32 ControllerId, const FForceFeedbackValues& Values) { } bool FSGXRTracker::SupportsForceFeedback(const int32 ControllerId) { // FIXME // For now it's true, but when we fall back to the hand tracking we might return false. (void) ControllerId; return true; } bool FSGXRTracker::IsGamepadAttached() const { return false; } FName FSGXRTracker::GetHandTrackerDeviceTypeName() const { return GetMotionControllerDeviceTypeName(); } bool FSGXRTracker::IsHandTrackingStateValid() const { const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bHandTrackingAvailable = Pimpl->bHandTrackingAvailable; const bool bAnyGloveConnected = FImpl::IsAnyGloveConnected(); const bool bHandTrackingStateValid = bAnyGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bHandTrackingAvailable); return bHandTrackingStateValid; } bool FSGXRTracker::GetKeypointState( const EControllerHand Hand, const EHandKeypoint Keypoint, FTransform& OutTransform, float& OutRadius) const { if (!IsHandTrackingStateValid()) { return false; } if (!Pimpl->CanOutputTrackingData(Hand)) { return false; } const FSGXRHandState& HandState{ Hand == EControllerHand::Left ? GetLeftHandState() : GetRightHandState() }; const bool bGotTransform = HandState.GetTransform(Keypoint, OutTransform); OutRadius = HandState.Radii[static_cast(Keypoint)]; if (bGotTransform) { if (!ensureAlwaysMsgf(Pimpl->XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } // Convert to UE world space. const FTransform& TrackingToWorldTransform{Pimpl->XRTrackingSystem->GetTrackingToWorldTransform()}; OutTransform *= TrackingToWorldTransform; } return bGotTransform; } bool FSGXRTracker::GetAllKeypointStates( const EControllerHand Hand, TArray& OutPositions, TArray& OutRotations, TArray& OutRadii) const { if (!IsHandTrackingStateValid()) { return false; } if (Hand != EControllerHand::Left && Hand != EControllerHand::Right) { return false; } if (!Pimpl->CanOutputTrackingData(Hand)) { return false; } const FSGXRHandState& HandState{ Hand == EControllerHand::Left ? GetLeftHandState() : GetRightHandState() }; if (!HandState.bReceivedJointPoses) { return false; } OutPositions.Empty(EHandKeypointCount); OutRotations.Empty(EHandKeypointCount); OutRadii.Empty(EHandKeypointCount); if (!ensureAlwaysMsgf(Pimpl->XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{Pimpl->XRTrackingSystem->GetTrackingToWorldTransform()}; for (int i = 0; i < EHandKeypointCount; ++i) { FTransform KeypointWorldTransform{HandState.KeypointTransforms[i] * TrackingToWorldTransform}; OutPositions.Add(KeypointWorldTransform.GetLocation()); OutRotations.Add(KeypointWorldTransform.GetRotation()); } for (int i = 0; i < EHandKeypointCount; ++i) { OutRadii.Add(HandState.Radii[i]); } return true; } FSGXRTracker::FImpl::FImpl(FSGXRTracker* InOwner) : bHandTrackingAvailable(false), XRCreateHandTrackerEXT(nullptr), XRDestroyHandTrackerEXT(nullptr), XRLocateHandJointsEXT(nullptr), XRTrackingSystem(nullptr), DeviceIndex(0), CurrentHandTrackingDataIndex(0), Owner(InOwner) { } FSGXRTracker::FImpl::~FImpl() = default; void FSGXRTracker::FImpl::BuildMotionSourceToKeypointMap() { ensureAlwaysMsgf(IsInGameThread(), TEXT("Is not in the game thread!")); if (!MotionSourceToKeypointMap.IsEmpty()) { return; } const bool bSupportLegacyControllerMotionSources = ShouldSupportLegacyControllerMotionSources(); // There is a motion source that corresponds to each hand keypoint of the form [Left|Right][Keypoint]. // Build a map so we can quickly translate from motion source FName to hand bone EHandKeypoint value. // We also have the option of using more specific motion sources of the form HandTracking[Left|Right][Keypoint]. // This is useful if one wishes to use hand tracking and controllers simultaneously. // We also may support more generic legacy motion sources, by default we do support this. const FName& Left(GetLeftHandTrackingMotionSourceName()); const FName& Right(GetRightHandTrackingMotionSourceName()); for (int32 KeypointIndex = 0; KeypointIndex < EHandKeypointCount; ++KeypointIndex) { const EHandKeypoint EnumValue = static_cast(KeypointIndex); const FName EnumName{GetHandKeypointEnumName(KeypointIndex)}; const FString EnumString{EnumName.ToString()}; FName LeftName(Left.ToString() + EnumString); FName RightName(Right.ToString() + EnumString); MotionSourceToKeypointMap.Add(MoveTemp(LeftName), FSGMotionSourceInfo(EnumValue, true)); MotionSourceToKeypointMap.Add(MoveTemp(RightName), FSGMotionSourceInfo(EnumValue, false)); } if (bSupportLegacyControllerMotionSources) { const FName& LeftLegacyControllerHandTrackingMotionSourceName{ GetLegacyControllerLeftHandTrackingMotionSourceName()}; const FName& RightLegacyControllerHandTrackingMotionSourceName{ GetLegacyControllerRightHandTrackingMotionSourceName()}; MotionSourceToKeypointMap.Add( LeftLegacyControllerHandTrackingMotionSourceName, FSGMotionSourceInfo(EHandKeypoint::Wrist, true)); MotionSourceToKeypointMap.Add( RightLegacyControllerHandTrackingMotionSourceName, FSGMotionSourceInfo(EHandKeypoint::Wrist, false)); } const FSGWristTrackingSettings& WristTrackingSettings{GetWristTrackingSettings()}; const FName LeftHandSGMotionSourceName{ GetControllerHandEnumName(WristTrackingSettings.LeftHandMotionSource) }; const FName RightHandSGMotionSourceName{ GetControllerHandEnumName(WristTrackingSettings.RightHandMotionSource) }; MotionSourceToKeypointMap.Add( LeftHandSGMotionSourceName, FSGMotionSourceInfo(EHandKeypoint::Wrist, true)); MotionSourceToKeypointMap.Add( RightHandSGMotionSourceName, FSGMotionSourceInfo(EHandKeypoint::Wrist, false)); } bool FSGXRTracker::FImpl::CanOutputTrackingData(const EControllerHand Hand) const { const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bIsHandTrackingAvailable = bHandTrackingAvailable; bool bGloveConnected = false; if (Hand == EControllerHand::Left) { bGloveConnected = IsLeftGloveConnected(); } else if (Hand == EControllerHand::Right) { bGloveConnected = IsRightGloveConnected(); } const bool bCanOutputTrackingData = bGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bIsHandTrackingAvailable); return bCanOutputTrackingData; } bool FSGXRTracker::FImpl::UpdateXRHandTrackingData( FSGXRHandState& OutHandState, const XrHandJointsLocateInfoEXT& LocateInfo) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); XR_ENSURE(XRLocateHandJointsEXT(OutHandState.HandTracker, &LocateInfo, &OutHandState.Locations)); OutHandState.bReceivedJointPoses = OutHandState.Locations.isActive == XR_TRUE; if (!OutHandState.bReceivedJointPoses) { return false; } static_assert( XR_HAND_JOINT_PALM_EXT == 0 && XR_HAND_JOINT_LITTLE_TIP_EXT == XR_HAND_JOINT_COUNT_EXT - 1, "XrHandJointEXT enum is not as assumed for the following loop!"); for (int j = 0; j < XR_HAND_JOINT_COUNT_EXT; ++j) { const XrHandJointLocationEXT& JointLocation{OutHandState.JointLocations[j]}; const XrPosef& Pose{JointLocation.pose}; FTransform Transform{ToFTransform(Pose, WorldToMetersScale)}; OutHandState.KeypointTransforms[j] = MoveTemp(Transform); OutHandState.Radii[j] = JointLocation.radius * WorldToMetersScale; // velocities would go here } return true; } bool FSGXRTracker::FImpl::GetControllerTransform( const FName MotionSource, const float WorldToMetersScale, FTransform& OutTransform, bool& bOutMotionSourceHandTracking) const { const FSGMotionSourceInfo* KeyPointInfoPtr{MotionSourceToKeypointMap.Find(MotionSource)}; if (!KeyPointInfoPtr) { return false; } const bool bRight = !KeyPointInfoPtr->Value; const bool bGlovePresent = (bRight && IsRightGloveConnected()) || (!bRight && IsLeftGloveConnected()); const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); if (!bGlovePresent && !bFallbackToHandTrackingIfNoGloveDetected) { return false; } static const FName OpenXRName{TEXT("OpenXR")}; static const FName OpenXRViveTrackerName{TEXT("OpenXRViveTracker")}; const FSGWristTrackingSettings& WristTrackingSettings{GetWristTrackingSettings()}; const EControllerHand PositionalTrackingHardwareHand = bRight ? WristTrackingSettings.RightHandMotionSource : WristTrackingSettings.LeftHandMotionSource; const FName& PositionalTrackingHardwareMotionSource{ GetControllerHandEnumName(PositionalTrackingHardwareHand) }; bool bTracked = false; if (MotionSource == PositionalTrackingHardwareMotionSource) { const TArray MotionControllers{ IModularFeatures::Get().GetModularFeatureImplementations( IMotionController::GetModularFeatureName()) }; bool bLookingForViveTrackers = false; if (WristTrackingSettings.TrackingHardware == ESGPositionalTrackingHardware::ViveTracker || WristTrackingSettings.TrackingHardware == ESGPositionalTrackingHardware::ViveFocus3WristTracker) { bLookingForViveTrackers = true; } else if (WristTrackingSettings.TrackingHardware == ESGPositionalTrackingHardware::None) { const bool bIsAHtcViveDevice = FSGHMDTracker::IsAHtcViveDevice(); if (bIsAHtcViveDevice) { bLookingForViveTrackers = true; } } for (const IMotionController* MotionController: MotionControllers) { if (!MotionController) { continue; } const ETrackingStatus TrackingStatus = MotionController->GetControllerTrackingStatus( DeviceIndex, PositionalTrackingHardwareMotionSource); if (TrackingStatus == ETrackingStatus::NotTracked) { continue; } if (bLookingForViveTrackers) { const FName DeviceTypeName{MotionController->GetMotionControllerDeviceTypeName()}; if (DeviceTypeName != OpenXRViveTrackerName) { continue; } } else { const FName DeviceTypeName{MotionController->GetMotionControllerDeviceTypeName()}; if (DeviceTypeName != OpenXRName) { continue; } } FRotator Orientation; FVector Position; const bool bGotTransform = MotionController->GetControllerOrientationAndPosition( DeviceIndex, PositionalTrackingHardwareMotionSource, Orientation, Position, WorldToMetersScale); if (bGotTransform) { OutTransform = FTransform{Orientation, Position}; bOutMotionSourceHandTracking = false; bTracked = true; break; } } } if (!bTracked && bFallbackToHandTrackingIfNoGloveDetected) { const FSGXRHandState& HandState{bRight ? Owner->GetRightHandState() : Owner->GetLeftHandState()}; if (!HandState.bReceivedJointPoses) { return false; } const EHandKeypoint KeyPoint = KeyPointInfoPtr->Key; const FTransform& ControllerTransform{ bRight ? GetRightHandState().GetTransform(KeyPoint) : GetLeftHandState().GetTransform(KeyPoint) }; OutTransform = ControllerTransform; bOutMotionSourceHandTracking = true; bTracked = true; } return bTracked; } bool FSGXRTracker::FImpl::GetWristTransform(const bool bRight, FTransform& OutTransform) const { const FSGWristTrackingSettings& WristTrackingSettings{GetWristTrackingSettings()}; const EControllerHand PositionalTrackingHardwareHand = bRight ? WristTrackingSettings.RightHandMotionSource : WristTrackingSettings.LeftHandMotionSource; const FName& MotionSource{ GetControllerHandEnumName(PositionalTrackingHardwareHand) }; if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); bool bMotionSourceHandTracking = false; FTransform ControllerTransform; const bool bGotTransform = GetControllerTransform( MotionSource, WorldToMetersScale, ControllerTransform, bMotionSourceHandTracking); if (bGotTransform) { if (bMotionSourceHandTracking) { OutTransform = MoveTemp(ControllerTransform); } else { const ESGPositionalTrackingHardware TrackingHardware = GetPositionalTrackingHardware(); if (TrackingHardware == ESGPositionalTrackingHardware::Custom) { if (bRight) { ControllerTransform.SetRotation( (ControllerTransform.GetRotation().Rotator() + WristTrackingSettings.TrackingHardwareRotationOffsetRightHand).Quaternion()); ControllerTransform.SetLocation( ControllerTransform.GetLocation() + WristTrackingSettings.TrackingHardwareLocationOffsetRightHand); } else { ControllerTransform.SetRotation( (ControllerTransform.GetRotation().Rotator() + WristTrackingSettings.TrackingHardwareRotationOffsetLeftHand).Quaternion()); ControllerTransform.SetLocation( ControllerTransform.GetLocation() + WristTrackingSettings.TrackingHardwareLocationOffsetLeftHand); } } const USGHapticGlove* Glove{GetGlove(bRight)}; if (!IsValid(Glove)) { return false; } FVector WristLocation; FRotator WristRotation; Glove->GetWristLocation( ControllerTransform.GetLocation(), ControllerTransform.GetRotation().Rotator(), TrackingHardware, WristLocation, WristRotation); OutTransform = FTransform{MoveTemp(WristRotation), MoveTemp(WristLocation)}; } } return bGotTransform; } bool FSGXRTracker::FImpl::UpdateSGJointData( FSGXRHandState& OutHandState, const EHandKeypoint Joint, const FTransform& JointTransform, const FTransform& WristTransform) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{XRTrackingSystem->GetTrackingToWorldTransform()}; const FTransform& WorldToTrackingTransform{TrackingToWorldTransform.Inverse()}; const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); const int32 JointIndex = static_cast(Joint); const FQuat WristRotation{WristTransform.GetRotation()}; const FVector WristLocation{WristTransform.GetLocation()}; const FQuat JointRotation{JointTransform.GetRotation()}; const FVector JointLocation{JointTransform.GetLocation()}; FTransform Transform{ WristRotation * JointRotation, WristLocation + (WristRotation * JointLocation) }; OutHandState.KeypointTransforms[JointIndex] = MoveTemp(Transform); OutHandState.Radii[JointIndex] = 0.01f * WorldToMetersScale; return true; } bool FSGXRTracker::FImpl::UpdateSGWristJointData( FSGXRHandState& OutHandState, const FTransform& WristTransform) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{XRTrackingSystem->GetTrackingToWorldTransform()}; const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); static constexpr EHandKeypoint Joint = EHandKeypoint::Wrist; static constexpr int32 JointIndex = static_cast(Joint); OutHandState.KeypointTransforms[JointIndex] = WristTransform; OutHandState.Radii[JointIndex] = 0.01f * WorldToMetersScale; return true; } bool FSGXRTracker::FImpl::UpdateSGPalmJointData( FSGXRHandState& OutHandState, const bool bRight, const FTransform& WristTransform) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } FTransform JointTransform{GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::Palm)}; if (!UpdateSGJointData(OutHandState, EHandKeypoint::Palm, MoveTemp(JointTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGThumbJointData( FSGXRHandState& OutHandState, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const { static constexpr int32 ThumbFingerIndex = 0; static constexpr int32 ThumbMetacarpalJointIndex = 0; static constexpr int32 ThumbProximalJointIndex = 1; static constexpr int32 ThumbDistalJointIndex = 2; static constexpr int32 ThumbTipJointIndex = 3; FTransform ThumbMetacarpalTransform{ JointRotations[ThumbFingerIndex][ThumbMetacarpalJointIndex], JointLocations[ThumbFingerIndex][ThumbMetacarpalJointIndex] }; FTransform ThumbProximalTransform{ JointRotations[ThumbFingerIndex][ThumbProximalJointIndex], JointLocations[ThumbFingerIndex][ThumbProximalJointIndex] }; FTransform ThumbDistalTransform{ JointRotations[ThumbFingerIndex][ThumbDistalJointIndex], JointLocations[ThumbFingerIndex][ThumbDistalJointIndex] }; FTransform ThumbTipTransform{ JointRotations[ThumbFingerIndex][ThumbTipJointIndex], JointLocations[ThumbFingerIndex][ThumbTipJointIndex] }; if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbMetacarpal, MoveTemp(ThumbMetacarpalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbProximal, MoveTemp(ThumbProximalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbDistal, MoveTemp(ThumbDistalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbTip, MoveTemp(ThumbTipTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGIndexJointData( FSGXRHandState& OutHandState, const bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const { static constexpr int32 IndexFingerIndex = 1; static constexpr int32 IndexProximalJointIndex = 0; static constexpr int32 IndexIntermediateJointIndex = 1; static constexpr int32 IndexDistalJointIndex = 2; static constexpr int32 IndexTipJointIndex = 3; FTransform IndexMetacarpalTransform{ GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::IndexMetacarpal) }; FTransform IndexProximalTransform{ JointRotations[IndexFingerIndex][IndexProximalJointIndex], JointLocations[IndexFingerIndex][IndexProximalJointIndex] }; FTransform IndexIntermediateTransform{ JointRotations[IndexFingerIndex][IndexIntermediateJointIndex], JointLocations[IndexFingerIndex][IndexIntermediateJointIndex] }; FTransform IndexDistalTransform{ JointRotations[IndexFingerIndex][IndexDistalJointIndex], JointLocations[IndexFingerIndex][IndexDistalJointIndex] }; FTransform IndexTipTransform{ JointRotations[IndexFingerIndex][IndexTipJointIndex], JointLocations[IndexFingerIndex][IndexTipJointIndex] }; if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexMetacarpal, MoveTemp(IndexMetacarpalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexProximal, MoveTemp(IndexProximalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexIntermediate, MoveTemp(IndexIntermediateTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexDistal, MoveTemp(IndexDistalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexTip, MoveTemp(IndexTipTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGMiddleJointData( FSGXRHandState& OutHandState, const bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const { static constexpr int32 MiddleFingerMiddle = 2; static constexpr int32 MiddleProximalJointMiddle = 0; static constexpr int32 MiddleIntermediateJointMiddle = 1; static constexpr int32 MiddleDistalJointMiddle = 2; static constexpr int32 MiddleTipJointMiddle = 3; FTransform MiddleMetacarpalTransform{ GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::MiddleMetacarpal) }; FTransform MiddleProximalTransform{ JointRotations[MiddleFingerMiddle][MiddleProximalJointMiddle], JointLocations[MiddleFingerMiddle][MiddleProximalJointMiddle] }; FTransform MiddleIntermediateTransform{ JointRotations[MiddleFingerMiddle][MiddleIntermediateJointMiddle], JointLocations[MiddleFingerMiddle][MiddleIntermediateJointMiddle] }; FTransform MiddleDistalTransform{ JointRotations[MiddleFingerMiddle][MiddleDistalJointMiddle], JointLocations[MiddleFingerMiddle][MiddleDistalJointMiddle] }; FTransform MiddleTipTransform{ JointRotations[MiddleFingerMiddle][MiddleTipJointMiddle], JointLocations[MiddleFingerMiddle][MiddleTipJointMiddle] }; if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleMetacarpal, MoveTemp(MiddleMetacarpalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleProximal, MoveTemp(MiddleProximalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleIntermediate, MoveTemp(MiddleIntermediateTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleDistal, MoveTemp(MiddleDistalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleTip, MoveTemp(MiddleTipTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGRingJointData( FSGXRHandState& OutHandState, const bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const { static constexpr int32 RingFingerRing = 3; static constexpr int32 RingProximalJointRing = 0; static constexpr int32 RingIntermediateJointRing = 1; static constexpr int32 RingDistalJointRing = 2; static constexpr int32 RingTipJointRing = 3; FTransform RingMetacarpalTransform{ GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::RingMetacarpal) }; FTransform RingProximalTransform{ JointRotations[RingFingerRing][RingProximalJointRing], JointLocations[RingFingerRing][RingProximalJointRing] }; FTransform RingIntermediateTransform{ JointRotations[RingFingerRing][RingIntermediateJointRing], JointLocations[RingFingerRing][RingIntermediateJointRing] }; FTransform RingDistalTransform{ JointRotations[RingFingerRing][RingDistalJointRing], JointLocations[RingFingerRing][RingDistalJointRing] }; FTransform RingTipTransform{ JointRotations[RingFingerRing][RingTipJointRing], JointLocations[RingFingerRing][RingTipJointRing] }; if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingMetacarpal, MoveTemp(RingMetacarpalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingProximal, MoveTemp(RingProximalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingIntermediate, MoveTemp(RingIntermediateTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingDistal, MoveTemp(RingDistalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingTip, MoveTemp(RingTipTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGLittleJointData( FSGXRHandState& OutHandState, const bool bRight, const TArray>& JointLocations, const TArray>& JointRotations, const FTransform& WristTransform) const { static constexpr int32 LittleFingerLittle = 4; static constexpr int32 LittleProximalJointLittle = 0; static constexpr int32 LittleIntermediateJointLittle = 1; static constexpr int32 LittleDistalJointLittle = 2; static constexpr int32 LittleTipJointLittle = 3; FTransform LittleMetacarpalTransform{ GetMeshWristChildBoneRefTransform(bRight, EHandKeypoint::LittleMetacarpal) }; FTransform LittleProximalTransform{ JointRotations[LittleFingerLittle][LittleProximalJointLittle], JointLocations[LittleFingerLittle][LittleProximalJointLittle] }; FTransform LittleIntermediateTransform{ JointRotations[LittleFingerLittle][LittleIntermediateJointLittle], JointLocations[LittleFingerLittle][LittleIntermediateJointLittle] }; FTransform LittleDistalTransform{ JointRotations[LittleFingerLittle][LittleDistalJointLittle], JointLocations[LittleFingerLittle][LittleDistalJointLittle] }; FTransform LittleTipTransform{ JointRotations[LittleFingerLittle][LittleTipJointLittle], JointLocations[LittleFingerLittle][LittleTipJointLittle] }; if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleMetacarpal, MoveTemp(LittleMetacarpalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleProximal, MoveTemp(LittleProximalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleIntermediate, MoveTemp(LittleIntermediateTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleDistal, MoveTemp(LittleDistalTransform), WristTransform)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleTip, MoveTemp(LittleTipTransform), WristTransform)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGHandState(const bool bRight, FSGXRHandState& OutHandState) const { const USGHandPose* HandPose{GetHandPose(bRight)}; if (!IsValid(HandPose)) { return false; } OutHandState.bReceivedJointPoses = true; const TArray> JointLocations{HandPose->GetJointPositions()}; const TArray> JointRotations{HandPose->GetJointRotationsQ()}; FTransform WristTransform; const bool bGotWristTransform = GetWristTransform(bRight, WristTransform); /// NOTE // Even if we fail to get the wrist transform, the SenseGlove data needs to make it to the FXRMotionControllerData! // So, we just ignore the bGotWristTransform results! Otherwise, even if bFallbackToHandTrackingIfNoGloveDetected // is disabled it will output the hand-tracking data. (void) bGotWristTransform; if (!UpdateSGWristJointData(OutHandState, WristTransform)) { return false; } if (!UpdateSGPalmJointData(OutHandState, bRight, WristTransform)) { return false; } if (!UpdateSGThumbJointData(OutHandState, JointLocations, JointRotations, WristTransform)) { return false; } if (!UpdateSGIndexJointData(OutHandState, bRight, JointLocations, JointRotations, WristTransform)) { return false; } if (!UpdateSGMiddleJointData(OutHandState, bRight, JointLocations, JointRotations, WristTransform)) { return false; } if (!UpdateSGRingJointData(OutHandState, bRight, JointLocations, JointRotations, WristTransform)) { return false; } if (!UpdateSGLittleJointData(OutHandState, bRight, JointLocations, JointRotations, WristTransform)) { return false; } return true; } void FSGXRTracker::FImplDeleter::operator()(const FImpl* P) const { delete P; }