/** * @file * * @author Mamadou Babaei * * @section LICENSE * * (The MIT License) * * Copyright (c) 2020 - 2024 SenseGlove * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * @section DESCRIPTION * * */ #include "SGTracking/SGXRTracker.h" #include "Containers/Array.h" #include "Containers/Map.h" #if WITH_EDITOR #include "Editor.h" #endif /* WITH_EDITOR */ #include "Engine/Engine.h" #include "Engine/SkeletalMesh.h" #include "Engine/World.h" #include "GameFramework/WorldSettings.h" #include "IOpenXRHMDModule.h" #include "IXRTrackingSystem.h" #include "OpenXRCore.h" #include "OpenXRHMD/Private/OpenXRHMD.h" #include "Runtime/Launch/Resources/Version.h" #if WITH_EDITOR #include "Subsystems/UnrealEditorSubsystem.h" #endif /* WITH_EDITOR */ #include "SGBackend/SGBackend.h" #include "SGCore/SGHapticGlove.h" #include "SGLog/SGLog.h" #include "SGSettings/SGSettings.h" #include "SGTracking/SGGloveTracker.h" #include "SGTracking/SGHMDTracker.h" #include "SGTracking/SGXRHandState.h" #include "SGTypes/SGCoreTypes.h" #include "SGTypes/SGTrackingTypes.h" #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 void EnumerateOpenXRApiLayers(TArray& OutProperties) { uint32_t Count = 0; XR_ENSURE(xrEnumerateApiLayerProperties(0, &Count, nullptr)); OutProperties.SetNum(Count); for (auto& Prop: OutProperties) { Prop = XrApiLayerProperties{XR_TYPE_API_LAYER_PROPERTIES}; } XR_ENSURE(xrEnumerateApiLayerProperties(Count, &Count, OutProperties.GetData())); } #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 3 */ struct FSGXRTracker::FImpl { /************************ * Typedefs ************************/ // bool true == left, false == right typedef TPair FSGMotionSourceInfo; /************************ * Static methods ************************/ static UWorld* GetWorld(); FORCEINLINE static bool ShouldFallbackToHandTrackingIfNoGloveDetected() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = !!Settings->GetTrackingSettings().bFallbackToHandTrackingIfNoGloveDetected; return bFallbackToHandTrackingIfNoGloveDetected; } FORCEINLINE static bool ShouldUseMoreSpecificMotionSourceNames() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bUseMoreSpecificMotionSourceNames = bFallbackToHandTrackingIfNoGloveDetected ? !!Settings->GetTrackingSettings().HandTrackingSettings.bUseMoreSpecificMotionSourceNames : false; return bUseMoreSpecificMotionSourceNames; } FORCEINLINE static bool ShouldSupportLegacyControllerMotionSources() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bSupportLegacyControllerMotionSources = bFallbackToHandTrackingIfNoGloveDetected ? !!Settings->GetTrackingSettings().HandTrackingSettings.bSupportLegacyControllerMotionSources : false; return bSupportLegacyControllerMotionSources; } FORCEINLINE static const FSGHandTrackingSettings& GetHandTrackingSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetTrackingSettings().HandTrackingSettings; } FORCEINLINE static const FSGVirtualHandTrackingSettings& GetVirtualHandTrackingSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetTrackingSettings().VirtualHandTrackingSettings; } FORCEINLINE static const FSGWristTrackingSettings& GetWristTrackingSettings() { const USGSettings* Settings{USGSettings::GetInstance()}; ensureAlwaysMsgf(Settings, TEXT("The settings subsystem has not been initialized!")); return Settings->GetTrackingSettings().WristTrackingSettings; } static ESGPositionalTrackingHardware GetPositionalTrackingHardware(); FORCEINLINE static USGHapticGlove* GetGlove(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetGlove(bRight) : nullptr; } FORCEINLINE static USGHapticGlove* GetGlove(const bool bRight) { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetGlove(bRight) : nullptr; } FORCEINLINE static USGHapticGlove* GetLeftGlove() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetLeftGlove() : nullptr; } FORCEINLINE static USGHapticGlove* GetRightGlove() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->GetRightGlove() : nullptr; } FORCEINLINE static bool IsGloveConnected(const EControllerHand Hand) { const bool bRight = Hand == EControllerHand::Right; const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsGloveConnected(bRight) : false; } FORCEINLINE static bool IsGloveConnected(const bool bRight) { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsGloveConnected(bRight) : false; } FORCEINLINE static bool IsLeftGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsLeftGloveConnected() : false; } FORCEINLINE static bool IsRightGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsRightGloveConnected() : false; } FORCEINLINE static bool IsAnyGloveConnected() { const USGGloveTracker* GloveTracker{USGGloveTracker::GetInstance()}; return IsValid(GloveTracker) ? GloveTracker->IsAnyGloveConnected() : false; } FORCEINLINE static const FName& GetLeftMeshBoneName(const int32 JointIndex) { const FName& Name{GetVirtualHandTrackingSettings().LeftHandBoneNames[JointIndex]}; return Name; } FORCEINLINE static const FName& GetLeftMeshBoneName(const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetLeftMeshBoneName(JointIndex)}; return Name; } FORCEINLINE static const FName& GetRightMeshBoneName(const int32 JointIndex) { const FName& Name{GetVirtualHandTrackingSettings().RightHandBoneNames[JointIndex]}; return Name; } FORCEINLINE static const FName& GetRightMeshBoneName(const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetRightMeshBoneName(JointIndex)}; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const bool bRight, const int32 Joint) { const FName& Name{ bRight ? GetRightMeshBoneName(Joint) : GetLeftMeshBoneName(Joint) }; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const bool bRight, const EHandKeypoint HandKeypoint) { const int32 JointIndex = static_cast(HandKeypoint); const FName& Name{GetMeshBoneName(bRight, JointIndex)}; return Name; } FORCEINLINE static const FName& GetMeshBoneName(const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FName& Name{GetMeshBoneName(bRight, JointIndex)}; return Name; } static const FTransform& GetRawMeshBoneRefTransform(bool bRight, const FName& BoneName); FORCEINLINE static const FTransform& GetRawMeshBoneRefTransform(const bool bRight, const int32 JointIndex) { const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform& Transform{GetRawMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static const FTransform& GetRawMeshBoneRefTransform(const bool bRight, const EHandKeypoint Keypoint) { const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform& Transform{GetRawMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static const FTransform& GetRawMeshBoneRefTransform(const EControllerHand Hand, const FName& BoneName) { const bool bRight = Hand == EControllerHand::Right; const FTransform& Transform{GetRawMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static const FTransform& GetRawMeshBoneRefTransform(const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform& Transform{GetRawMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static const FTransform& GetRawMeshBoneRefTransform( const EControllerHand Hand, const EHandKeypoint Keypoint) { const bool bRight = Hand == EControllerHand::Right; const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform& Transform{GetRawMeshBoneRefTransform(bRight, BoneName)}; return Transform; } static FTransform GetMeshBoneRefTransform(bool bRight, const FName& BoneName); FORCEINLINE static FTransform GetMeshBoneRefTransform(const bool bRight, const int32 JointIndex) { const FName& BoneName{GetMeshBoneName(bRight, JointIndex)}; const FTransform Transform{GetMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshBoneRefTransform(const bool bRight, const EHandKeypoint Keypoint) { const FName& BoneName{GetMeshBoneName(bRight, Keypoint)}; const FTransform Transform{GetMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshBoneRefTransform(const EControllerHand Hand, const FName& BoneName) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshBoneRefTransform(bRight, BoneName)}; return Transform; } FORCEINLINE static FTransform GetMeshBoneRefTransform(const EControllerHand Hand, const int32 JointIndex) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshBoneRefTransform(bRight, JointIndex)}; return Transform; } FORCEINLINE static FTransform GetMeshBoneRefTransform(const EControllerHand Hand, const EHandKeypoint Keypoint) { const bool bRight = Hand == EControllerHand::Right; const FTransform Transform{GetMeshBoneRefTransform(bRight, Keypoint)}; return Transform; } /************************ * Member variables ************************/ uint8 bHandTrackingAvailable : 1; PFN_xrCreateHandTrackerEXT XRCreateHandTrackerEXT; PFN_xrDestroyHandTrackerEXT XRDestroyHandTrackerEXT; PFN_xrLocateHandJointsEXT XRLocateHandJointsEXT; IXRTrackingSystem* XRTrackingSystem; TSharedPtr MessageHandler; int32 DeviceIndex; int32 CurrentHandTrackingDataIndex; TArray BoneParents; TArray BoneKeypoints; TMap MotionSourceToKeypointMap; FSGXRHandState HandStates[2]; TArray LeftAnimationTransforms; TArray RightAnimationTransforms; /************************ * Owner object ************************/ FSGXRTracker* Owner; /************************ * Constructor / Destructor ************************/ explicit FImpl(FSGXRTracker* InOwner); ~FImpl(); /************************ * Default copy constructor & copy assignment operator ************************/ FImpl(const FImpl& Rhs) = delete; FImpl& operator=(const FImpl& Rhs) = delete; /************************ * Methods ************************/ FSGXRHandState& GetLeftHandState(); FSGXRHandState& GetRightHandState(); void AddKeys(); void BuildMotionSourceToKeypointMap(); bool CanOutputTrackingData(EControllerHand Hand) const; bool UpdateXRHandTrackingData(FSGXRHandState& OutHandState, const XrHandJointsLocateInfoEXT& LocateInfo) const; bool GetOpenXRHandTrackingWristLocationAndRotation(EControllerHand DeviceHand, FVector& OutWristLocation, FQuat& OutWristRotation) const; bool GetOpenXRWristLocationAndRotation(const EControllerHand DeviceHand, const IMotionController* MotionController, const FName& MotionSource, FVector& OutWristLocation, FQuat& OutWristRotation) const; bool GetWristLocationAndRotation(EControllerHand DeviceHand, FVector& OutWristLocation, FQuat& OutWristRotation, bool& bOutMotionSourceHandTracking) const; bool UpdateSGJointData(FSGXRHandState& OutHandState, const EHandKeypoint Joint, const FVector& JointLocation, const FQuat& JointRotation, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGWristJointData(FSGXRHandState& OutHandState, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGPalmJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGThumbJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGIndexJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGMiddleJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGRingJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGLittleJointData( FSGXRHandState& OutHandState, EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, bool bMotionSourceHandTracking) const; bool UpdateSGHandState(const EControllerHand DeviceHand, FSGXRHandState& OutHandState) const; }; UWorld* FSGXRTracker::FImpl::GetWorld() { UWorld* World{nullptr}; #if WITH_EDITOR if (GIsEditor) { if (IsValid(GEditor)) { const bool bIsPIE = GPlayInEditorID != -1; if (!bIsPIE) { const FWorldContext* WorldContext{GEditor->GetPIEWorldContext(1)}; if (WorldContext) { World = WorldContext->World(); } } else { const FWorldContext* WorldContext{GEditor->GetPIEWorldContext(GPlayInEditorID)}; if (WorldContext) { World = WorldContext->World(); } } if (!IsValid(World)) { UUnrealEditorSubsystem* UnrealEditorSubsystem{GEditor->GetEditorSubsystem()}; if (IsValid(UnrealEditorSubsystem)) { World = UnrealEditorSubsystem->GetEditorWorld(); } } } if (!IsValid(World) && IsValid(GEngine)) { const UGameViewportClient* Viewport{GEngine->GameViewport}; if (IsValid(Viewport)) { World = Viewport->GetWorld(); } } } else { if (IsValid(GEngine)) { World = GEngine->GetCurrentPlayWorld(nullptr); } } #else /* WITH_EDITOR */ if (IsValid(GEngine)) { World = GEngine->GetCurrentPlayWorld(nullptr); } #endif /* WITH_EDITOR */ if (!IsValid(World) && IsValid(GEngine)) { UEngineSubsystem* EngineSubsystem{GEngine->GetEngineSubsystem()}; if (IsValid(EngineSubsystem)) { World = EngineSubsystem->GetWorld(); } } return World; } ESGPositionalTrackingHardware FSGXRTracker::FImpl::GetPositionalTrackingHardware() { ESGPositionalTrackingHardware TrackerHardware = GetWristTrackingSettings().TrackingHardware; if (TrackerHardware == ESGPositionalTrackingHardware::None) { const ESGHeadMountedDisplayDevice HmdDevice = FSGHMDTracker::GetDeviceType(); switch (HmdDevice) { case ESGHeadMountedDisplayDevice::MetaQuest2: TrackerHardware = ESGPositionalTrackingHardware::Quest2Controller; break; case ESGHeadMountedDisplayDevice::MetaQuestPro: TrackerHardware = ESGPositionalTrackingHardware::QuestProController; break; case ESGHeadMountedDisplayDevice::MetaQuest3: TrackerHardware = ESGPositionalTrackingHardware::Quest3Controller; break; case ESGHeadMountedDisplayDevice::HtcVivePro: TrackerHardware = ESGPositionalTrackingHardware::ViveTracker; break; case ESGHeadMountedDisplayDevice::HtcViveFocus3: // Fall through case ESGHeadMountedDisplayDevice::HtcViveXRElite: TrackerHardware = ESGPositionalTrackingHardware::ViveFocus3WristTracker; break; default: TrackerHardware = ESGPositionalTrackingHardware::Custom; break; } } return TrackerHardware; } const FTransform& FSGXRTracker::FImpl::GetRawMeshBoneRefTransform(const bool bRight, const FName& BoneName) { const TSoftObjectPtr& HandMesh{ bRight ? GetVirtualHandTrackingSettings().LeftHandReferenceMesh : GetVirtualHandTrackingSettings().RightHandReferenceMesh }; if (HandMesh.IsValid()) { const FReferenceSkeleton& ReferenceSkeleton{HandMesh->GetRefSkeleton()}; const int32 BoneIndex = ReferenceSkeleton.FindBoneIndex(BoneName); if (BoneIndex == INDEX_NONE) { SGLOG_WARNING(FString::Printf(TEXT("Could not find a bone index for the '%s' joint!"), *BoneName.ToString())); } const TArray Transforms{ReferenceSkeleton.GetRefBonePose()}; ensureAlwaysMsgf(Transforms.Num() > 0 && Transforms.Num() > BoneIndex, TEXT("Invalid bone index '%d' or no transforms found!"), BoneIndex); const FTransform& Transform{Transforms[BoneIndex]}; return Transform; } const FSGVirtualHandTrackingSettings& Settings{GetVirtualHandTrackingSettings()}; ensureAlwaysMsgf( bRight ? Settings.RightHandDefaultReferenceBoneTransforms.Contains(BoneName) : Settings.LeftHandDefaultReferenceBoneTransforms.Contains(BoneName), TEXT("Invalid bone name '%s'!"), *BoneName.ToString()); const FTransform& Transform{ bRight ? Settings.RightHandDefaultReferenceBoneTransforms[BoneName] : Settings.LeftHandDefaultReferenceBoneTransforms[BoneName] }; return Transform; } FTransform FSGXRTracker::FImpl::GetMeshBoneRefTransform(const bool bRight, const FName& BoneName) { const FQuat RootRotationCorrection{GetVirtualHandTrackingSettings().HandRootBoneRotationCorrection.Quaternion()}; FTransform RootBoneRefTransform{GetRawMeshBoneRefTransform(bRight, EHandKeypoint::Wrist)}; RootBoneRefTransform.SetRotation(RootRotationCorrection * RootBoneRefTransform.GetRotation()); const FTransform Transform{GetRawMeshBoneRefTransform(bRight, BoneName) * RootBoneRefTransform}; return Transform; } FName FSGXRTracker::ParseEOpenXRHandKeypointEnumName(const FName& EnumName) { static const int32 EnumNameLength = FString{TEXT("EHandKeypoint::")}.Len(); const FString EnumString{EnumName.ToString()}; const FName ParsedName{*EnumString.Right(EnumString.Len() - EnumNameLength)}; return ParsedName; } FSGXRTracker::FSGXRTracker(const TSharedRef& InMessageHandler) : Pimpl(TUniquePtr(new FImpl{this}, PimplDeleter)) { Pimpl->MessageHandler = InMessageHandler; // Register the modular features manually. IModularFeatures::Get().RegisterModularFeature(IHandTracker::GetModularFeatureName(), static_cast(this)); IModularFeatures::Get().RegisterModularFeature(IMotionController::GetModularFeatureName(), static_cast(this)); IModularFeatures::Get().RegisterModularFeature(IOpenXRExtensionPlugin::GetModularFeatureName(), static_cast(this)); Pimpl->AddKeys(); // We're implicitly requiring that the OpenXRPlugin has been loaded and initialized at this point. if (!IOpenXRHMDModule::Get().IsAvailable()) { SGLOG_FATAL("The OpenXRHMD plugin isn't available!"); ensureAlwaysMsgf(false, TEXT("The OpenXRHMD plugin isn't available!")); } } FSGXRTracker::FSGXRTracker(FSGXRTracker&& Rhs) SG_NOEXCEPT = default; FSGXRTracker::~FSGXRTracker() = default; FSGXRTracker& FSGXRTracker::operator=(FSGXRTracker&& Rhs) SG_NOEXCEPT = default; bool FSGXRTracker::IsHandTrackingSupportedByDevice() const { const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bAnyGloveConnected = FImpl::IsAnyGloveConnected(); const bool bHandTrackingAvailable = !!Pimpl->bHandTrackingAvailable; const bool bHandTrackingSupportedByDevice = bAnyGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bHandTrackingAvailable); return bHandTrackingSupportedByDevice; } const FSGXRHandState& FSGXRTracker::GetLeftHandState() const { return Pimpl->GetLeftHandState(); } const FSGXRHandState& FSGXRTracker::GetRightHandState() const { return Pimpl->GetRightHandState(); } FString FSGXRTracker::GetDisplayName() { static const FString Name{TEXT("SenseGloveTracking")}; return Name; } bool FSGXRTracker::GetRequiredExtensions(TArray& OutExtensions) { TArray Properties; EnumerateOpenXRApiLayers(Properties); // Some API layers can crash the loader when enabled, if they're present we shouldn't enable the extension. for (const XrApiLayerProperties& Layer: Properties) { if (FCStringAnsi::Strstr(Layer.layerName, "XR_APILAYER_VIVE_hand_tracking") && Layer.layerVersion <= 1) { return false; } } OutExtensions.Add("XR_EXT_hand_tracking"); return true; } const void* FSGXRTracker::OnGetSystem(const XrInstance InInstance, const void* InNext) { // Store the extension open xr calls to member function pointers for convenient use. XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrCreateHandTrackerEXT", (PFN_xrVoidFunction*)&Pimpl->XRCreateHandTrackerEXT)); XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrDestroyHandTrackerEXT", (PFN_xrVoidFunction*)&Pimpl->XRDestroyHandTrackerEXT)); XR_ENSURE(xrGetInstanceProcAddr(InInstance, "xrLocateHandJointsEXT", (PFN_xrVoidFunction*)&Pimpl->XRLocateHandJointsEXT)); return InNext; } const void* FSGXRTracker::OnCreateSession(const XrInstance InInstance, const XrSystemId InSystem, const void* InNext) { // Need to wait until the EHandKeypoint enum has been loaded, so we do this here rather than in the constructor // which runs too early. Pimpl->BuildMotionSourceToKeypointMap(); XrSystemHandTrackingPropertiesEXT HandTrackingSystemProperties{ XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT}; XrSystemProperties systemProperties{XR_TYPE_SYSTEM_PROPERTIES, &HandTrackingSystemProperties}; XR_ENSURE(xrGetSystemProperties(InInstance, InSystem, &systemProperties)); Pimpl->bHandTrackingAvailable = HandTrackingSystemProperties.supportsHandTracking == XR_TRUE; return InNext; } const void* FSGXRTracker::OnBeginSession(const XrSession InSession, const void* InNext) { const bool bHandTrackingAvailable = !!Pimpl->bHandTrackingAvailable; if (bHandTrackingAvailable) { static FName SystemName(TEXT("OpenXR")); if (IsValid(GEngine) && GEngine->XRSystem.IsValid() && (GEngine->XRSystem->GetSystemName() == SystemName)) { Pimpl->XRTrackingSystem = GEngine->XRSystem.Get(); } // Create a hand tracker for the left hand that tracks default set of hand joints. FSGXRHandState& LeftHandState{Pimpl->GetLeftHandState()}; { XrHandTrackerCreateInfoEXT CreateInfo{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT}; CreateInfo.hand = XR_HAND_LEFT_EXT; CreateInfo.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT; XR_ENSURE(Pimpl->XRCreateHandTrackerEXT(InSession, &CreateInfo, &LeftHandState.HandTracker)); } // Create a hand tracker for the right hand that tracks default set of hand joints. FSGXRHandState& RightHandState{Pimpl->GetRightHandState()}; { XrHandTrackerCreateInfoEXT CreateInfo{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT}; CreateInfo.hand = XR_HAND_RIGHT_EXT; CreateInfo.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT; XR_ENSURE(Pimpl->XRCreateHandTrackerEXT(InSession, &CreateInfo, &RightHandState.HandTracker)); } } return InNext; } void FSGXRTracker::UpdateDeviceLocations( const XrSession InSession, const XrTime DisplayTime, const XrSpace TrackingSpace) { if (!IsHandTrackingStateValid()) { return; } XrHandJointsLocateInfoEXT LocateInfo{XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT}; LocateInfo.baseSpace = TrackingSpace; LocateInfo.time = DisplayTime; const bool bLeftGloveConnected = FImpl::IsLeftGloveConnected(); if (bLeftGloveConnected) { /// NOTE /// It's necessary to call the UpdateXRHandTrackingData first for the wrist tracking to work properly when the /// motion controllers are not active and we rely on the OpenXRHandTracking data for wrist tracking. /// See the NOTE inside the FSGXRTracker::FImpl::GetWristLocationAndRotation method. Pimpl->UpdateXRHandTrackingData(Pimpl->HandStates[0], LocateInfo); Pimpl->UpdateSGHandState(EControllerHand::Left, Pimpl->HandStates[0]); } else { Pimpl->UpdateXRHandTrackingData(Pimpl->HandStates[0], LocateInfo); } const bool bRightGloveConnected = FImpl::IsRightGloveConnected(); if (bRightGloveConnected) { /// NOTE /// It's necessary to call the UpdateXRHandTrackingData first for the wrist tracking to work properly when the /// motion controllers are not active and we rely on the OpenXRHandTracking data for wrist tracking. /// See the NOTE inside the FSGXRTracker::FImpl::GetWristLocationAndRotation method. Pimpl->UpdateXRHandTrackingData(Pimpl->HandStates[1], LocateInfo); Pimpl->UpdateSGHandState(EControllerHand::Right, Pimpl->HandStates[1]); } else { Pimpl->UpdateXRHandTrackingData(Pimpl->HandStates[1], LocateInfo); } } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 bool FSGXRTracker::GetControllerOrientationAndPosition(const int32 ControllerIndex, const FName MotionSource, FRotator& OutOrientation, FVector& OutPosition, const float WorldToMetersScale) const #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ bool FSGXRTracker::GetControllerOrientationAndPosition(const int32 ControllerIndex, const EControllerHand DeviceHand, FRotator& OutOrientation, FVector& OutPosition, const float WorldToMetersScale) const #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ { if (!IsHandTrackingStateValid()) { return false; } // Hand tracking currently does not support late update. Current hand tracking systems have latency that make it // pointless. if (!IsInGameThread()) { return false; } bool bTracked = false; if (ControllerIndex == Pimpl->DeviceIndex) { FTransform ControllerTransform{FTransform::Identity}; #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 const FImpl::FSGMotionSourceInfo* KeyPointInfoPtr{Pimpl->MotionSourceToKeypointMap.Find(MotionSource)}; if (KeyPointInfoPtr) #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ if (GetControllerTrackingStatus(ControllerIndex, DeviceHand) != ETrackingStatus::NotTracked) #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ { #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 const EHandKeypoint KeyPoint = KeyPointInfoPtr->Key; const bool bIsLeft = KeyPointInfoPtr->Value; #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ const bool bIsLeft = DeviceHand == EControllerHand::Left; #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ if (bIsLeft) { #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 ControllerTransform = GetLeftHandState().GetTransform(KeyPoint); #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ ControllerTransform = GetLeftHandState().GetTransform(EHandKeypoint::Wrist); #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ bTracked = !!GetLeftHandState().bReceivedJointPoses; } else { #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 ControllerTransform = GetRightHandState().GetTransform(KeyPoint); #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ ControllerTransform = GetRightHandState().GetTransform(EHandKeypoint::Wrist); #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ bTracked = !!GetRightHandState().bReceivedJointPoses; } } if (bTracked) { OutPosition = ControllerTransform.GetLocation(); OutOrientation = ControllerTransform.GetRotation().Rotator(); } else { static const FName OpenXRName{TEXT("OpenXR")}; #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 const bool bUseMoreSpecificMotionSourceNames = FImpl::ShouldUseMoreSpecificMotionSourceNames(); static const FName LeftName(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingLeft") : TEXT("Left")); static const FName RightName(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingRight") : TEXT("Right")); FName MotionSource; if (DeviceHand == EControllerHand::Left) { MotionSource = LeftName; } else if (DeviceHand == EControllerHand::Right) { MotionSource = RightName; } #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION <= 2 */ const TArray MotionControllers{ IModularFeatures::Get().GetModularFeatureImplementations( IMotionController::GetModularFeatureName()) }; for (const IMotionController* MotionController: MotionControllers) { if (!MotionController) { continue; } const FName DeviceTypeName{MotionController->GetMotionControllerDeviceTypeName()}; if (DeviceTypeName != OpenXRName) { continue; } const ETrackingStatus TrackingStatus = MotionController->GetControllerTrackingStatus(0, MotionSource); if (TrackingStatus != ETrackingStatus::Tracked) { continue; } const bool bGotTransform = MotionController->GetControllerOrientationAndPosition( ControllerIndex, MotionSource, OutOrientation, OutPosition, WorldToMetersScale); if (bGotTransform) { bTracked = true; break; } } } } return bTracked; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 ETrackingStatus FSGXRTracker::GetControllerTrackingStatus(const int32 ControllerIndex, const FName MotionSource) const #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ ETrackingStatus FSGXRTracker::GetControllerTrackingStatus(const int32 ControllerIndex, const EControllerHand DeviceHand) const #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ { if (!IsHandTrackingStateValid()) { return ETrackingStatus::NotTracked; } #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 const FImpl::FSGMotionSourceInfo* KeyPointInfoPtr{Pimpl->MotionSourceToKeypointMap.Find(MotionSource)}; if (KeyPointInfoPtr) { const bool bIsLeft = KeyPointInfoPtr->Value; #else /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ const bool bIsLeft = DeviceHand == EControllerHand::Left; #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ if ((bIsLeft && FImpl::IsLeftGloveConnected()) || (!bIsLeft && FImpl::IsRightGloveConnected())) { return ETrackingStatus::Tracked; } const FSGXRHandState& HandState{bIsLeft ? GetLeftHandState() : GetRightHandState()}; return !!HandState.bReceivedJointPoses ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked; #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 } #endif /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 3 */ return ETrackingStatus::NotTracked; } FName FSGXRTracker::GetMotionControllerDeviceTypeName() const { const static FName DefaultName(TEXT("OpenXRHandTracking")); return DefaultName; } void FSGXRTracker::EnumerateSources(TArray& SourcesOut) const { ensureAlwaysMsgf(IsInGameThread(), TEXT("Is not in the game thread!")); const bool bUseMoreSpecificMotionSourceNames = FImpl::ShouldUseMoreSpecificMotionSourceNames(); SourcesOut.Reserve(SourcesOut.Num() + (EHandKeypointCount * 2)); const UEnum* EnumPtr{ FindObject(nullptr, TEXT("/Script/HeadMountedDisplay.EHandKeypoint"), true) }; ensureAlwaysMsgf(EnumPtr, TEXT("Failed to find the HMD hand keypoint!")); const FString Left(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingLeft") : TEXT("Left")); const FString Right(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingRight") : TEXT("Right")); for (int32 Keypoint = 0; Keypoint < EHandKeypointCount; Keypoint++) { const FString EnumString{ParseEOpenXRHandKeypointEnumName(EnumPtr->GetNameByValue(Keypoint)).ToString()}; const FString StringLeft{FString::Printf(TEXT("%s%s"), *Left, *EnumString)}; const FString StringRight{FString::Printf(TEXT("%s%s"), *Right, *EnumString)}; FName SourceL(*(StringLeft)); FName SourceR(*(StringRight)); SourcesOut.Add(MoveTemp(SourceL)); SourcesOut.Add(MoveTemp(SourceR)); } } void FSGXRTracker::Tick(const float DeltaTime) { } void FSGXRTracker::SendControllerEvents() { } void FSGXRTracker::SetMessageHandler(const TSharedRef& InMessageHandler) { Pimpl->MessageHandler = InMessageHandler; } bool FSGXRTracker::Exec(UWorld* InWorld, const TCHAR* Cmd, FOutputDevice& Ar) { return false; } void FSGXRTracker::SetChannelValue( const int32 ControllerId, const FForceFeedbackChannelType ChannelType, const float Value) { } void FSGXRTracker::SetChannelValues(const int32 ControllerId, const FForceFeedbackValues& Values) { } bool FSGXRTracker::SupportsForceFeedback(const int32 ControllerId) { // FIXME // For now it's true, but when we fall back to the hand tracking we might return false. (void) ControllerId; return true; } bool FSGXRTracker::IsGamepadAttached() const { return false; } FName FSGXRTracker::GetHandTrackerDeviceTypeName() const { return GetMotionControllerDeviceTypeName(); } bool FSGXRTracker::IsHandTrackingStateValid() const { const bool bFallbackToHandTrackingIfNoGloveDetected = FImpl::ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bHandTrackingAvailable = !!Pimpl->bHandTrackingAvailable; const bool bAnyGloveConnected = FImpl::IsAnyGloveConnected(); const bool bHandTrackingStateValid = bAnyGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bHandTrackingAvailable); return bHandTrackingStateValid; } bool FSGXRTracker::GetKeypointState( const EControllerHand Hand, const EHandKeypoint Keypoint, FTransform& OutTransform, float& OutRadius) const { if (!IsHandTrackingStateValid()) { return false; } if (!Pimpl->CanOutputTrackingData(Hand)) { return false; } bool bGotTransform = false; /// NOTE // Currently there is no OpenXR input simulation implementation. Maybe we will do that soon though? Leaving this // for reference for now. //#if WITH_INPUT_SIMULATION // if (auto* InputSim = UOpenXRInputSimulationEngineSubsystem::GetInputSimulationIfEnabled()) // { // bGotTransform = InputSim->GetHandJointTransform(Hand, Keypoint, OutTransform); // OutRadius = FSGXRHandState.Radii[static_cast(Keypoint)]; // } // else //#endif { const FSGXRHandState& HandState{ Hand == EControllerHand::Left ? GetLeftHandState() : GetRightHandState() }; bGotTransform = HandState.GetTransform(Keypoint, OutTransform); OutRadius = HandState.Radii[static_cast(Keypoint)]; } if (bGotTransform) { if (!ensureAlwaysMsgf(Pimpl->XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } // Convert to UE world space. const FTransform& TrackingToWorldTransform{Pimpl->XRTrackingSystem->GetTrackingToWorldTransform()}; OutTransform *= TrackingToWorldTransform; } return bGotTransform; } bool FSGXRTracker::GetAllKeypointStates( const EControllerHand Hand, TArray& OutPositions, TArray& OutRotations, TArray& OutRadii) const { if (!IsHandTrackingStateValid()) { return false; } if (Hand != EControllerHand::Left && Hand != EControllerHand::Right) { return false; } if (!Pimpl->CanOutputTrackingData(Hand)) { return false; } const FSGXRHandState& HandState{ Hand == EControllerHand::Left ? GetLeftHandState() : GetRightHandState() }; if (!HandState.bReceivedJointPoses) { return false; } OutPositions.Empty(EHandKeypointCount); OutRotations.Empty(EHandKeypointCount); OutRadii.Empty(EHandKeypointCount); if (!ensureAlwaysMsgf(Pimpl->XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{Pimpl->XRTrackingSystem->GetTrackingToWorldTransform()}; for (int i = 0; i < EHandKeypointCount; ++i) { FTransform KeypointWorldTransform{HandState.KeypointTransforms[i] * TrackingToWorldTransform}; OutPositions.Add(KeypointWorldTransform.GetLocation()); OutRotations.Add(KeypointWorldTransform.GetRotation()); } for (int i = 0; i < EHandKeypointCount; ++i) { OutRadii.Add(HandState.Radii[i]); } return true; } FSGXRTracker::FImpl::FImpl(FSGXRTracker* InOwner) : bHandTrackingAvailable(false), XRCreateHandTrackerEXT(nullptr), XRDestroyHandTrackerEXT(nullptr), XRLocateHandJointsEXT(nullptr), XRTrackingSystem(nullptr), DeviceIndex(0), CurrentHandTrackingDataIndex(0), Owner(InOwner) { } FSGXRTracker::FImpl::~FImpl() = default; FSGXRHandState& FSGXRTracker::FImpl::GetLeftHandState() { return HandStates[0]; } FSGXRHandState& FSGXRTracker::FImpl::GetRightHandState() { return HandStates[1]; } void FSGXRTracker::FImpl::AddKeys() { } void FSGXRTracker::FImpl::BuildMotionSourceToKeypointMap() { if (!MotionSourceToKeypointMap.IsEmpty()) { return; } const bool bUseMoreSpecificMotionSourceNames = ShouldUseMoreSpecificMotionSourceNames(); const bool bSupportLegacyControllerMotionSources = ShouldSupportLegacyControllerMotionSources(); // There is a motion source that corresponds to each hand keypoint of the form [Left|Right][Keypoint]. // Build a map so we can quickly translate from motion source FName to hand bone EHandKeypoint value. // We also have the option of using more specific motion sources of the form HandTracking[Left|Right][Keypoint]. // This is useful if one wishes to use hand tracking and controllers simultaneously. // We also may support more generic legacy motion sources, by default we do support this. const UEnum* EnumPtr{ FindObject(nullptr, TEXT("/Script/HeadMountedDisplay.EHandKeypoint"), true) }; ensureAlwaysMsgf(EnumPtr, TEXT("Failed to find the HMD hand keypoint!")); ensureAlwaysMsgf(IsInGameThread(), TEXT("Is not in the game thread!")); const FString Left(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingLeft") : TEXT("Left")); const FString Right(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingRight") : TEXT("Right")); for (int64 e = 0; e < EHandKeypointCount; ++e) { const EHandKeypoint EnumValue = static_cast(e); const FString EnumName{EnumPtr->GetNameStringByValue(e)}; FName LeftName(Left + EnumName); FName RightName(Right + EnumName); MotionSourceToKeypointMap.Add(MoveTemp(LeftName), FSGMotionSourceInfo(EnumValue, true)); MotionSourceToKeypointMap.Add(MoveTemp(RightName), FSGMotionSourceInfo(EnumValue, false)); } if (bSupportLegacyControllerMotionSources) { MotionSourceToKeypointMap.Add(FName("Left"), FSGMotionSourceInfo(EHandKeypoint::Wrist, true)); MotionSourceToKeypointMap.Add(FName("Right"), FSGMotionSourceInfo(EHandKeypoint::Wrist, false)); } } bool FSGXRTracker::FImpl::CanOutputTrackingData(const EControllerHand Hand) const { const bool bFallbackToHandTrackingIfNoGloveDetected = ShouldFallbackToHandTrackingIfNoGloveDetected(); const bool bIsHandTrackingAvailable = !!bHandTrackingAvailable; bool bGloveConnected = false; if (Hand == EControllerHand::Left) { bGloveConnected = IsLeftGloveConnected(); } else if (Hand == EControllerHand::Right) { bGloveConnected = IsRightGloveConnected(); } const bool bCanOutputTrackingData = bGloveConnected || (bFallbackToHandTrackingIfNoGloveDetected && bIsHandTrackingAvailable); return bCanOutputTrackingData; } bool FSGXRTracker::FImpl::UpdateXRHandTrackingData( FSGXRHandState& OutHandState, const XrHandJointsLocateInfoEXT& LocateInfo) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); XR_ENSURE(XRLocateHandJointsEXT(OutHandState.HandTracker, &LocateInfo, &OutHandState.Locations)); OutHandState.bReceivedJointPoses = OutHandState.Locations.isActive == XR_TRUE; if (!!OutHandState.bReceivedJointPoses) { static_assert( XR_HAND_JOINT_PALM_EXT == 0 && XR_HAND_JOINT_LITTLE_TIP_EXT == XR_HAND_JOINT_COUNT_EXT - 1, "XrHandJointEXT enum is not as assumed for the following loop!"); for (int j = 0; j < XR_HAND_JOINT_COUNT_EXT; ++j) { const XrHandJointLocationEXT& JointLocation{OutHandState.JointLocations[j]}; const XrPosef& Pose{JointLocation.pose}; FTransform Transform{ToFTransform(Pose, WorldToMetersScale)}; OutHandState.KeypointTransforms[j] = MoveTemp(Transform); OutHandState.Radii[j] = JointLocation.radius * WorldToMetersScale; // velocities would go here } return true; } return false; } bool FSGXRTracker::FImpl::GetOpenXRHandTrackingWristLocationAndRotation( const EControllerHand DeviceHand, FVector& OutWristLocation, FQuat& OutWristRotation) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } UWorld* World{GetWorld()}; FXRMotionControllerData MotionControllerData; XRTrackingSystem->GetMotionControllerData(World, DeviceHand, MotionControllerData); if (MotionControllerData.bValid) { static constexpr int32_t WristJointIndex = static_cast(EHandKeypoint::Wrist); OutWristLocation = MotionControllerData.HandKeyPositions[WristJointIndex]; OutWristRotation = MotionControllerData.HandKeyRotations[WristJointIndex]; return true; } return false; } bool FSGXRTracker::FImpl::GetOpenXRWristLocationAndRotation( const EControllerHand DeviceHand, const IMotionController* MotionController, const FName& MotionSource, FVector& OutWristLocation, FQuat& OutWristRotation) const { static constexpr int32 CurrentPlayerControllerIndex = 0; const UWorld* World{GetWorld()}; const float WorldToMetersScale = World ? World->GetWorldSettings()->WorldToMeters : 100.0f; FVector TrackerLocation; FRotator TrackerRotation; const bool bGotTransform = MotionController->GetControllerOrientationAndPosition( CurrentPlayerControllerIndex, MotionSource, TrackerRotation, TrackerLocation, WorldToMetersScale); if (bGotTransform) { const bool bIsRight = DeviceHand == EControllerHand::Right; const ESGPositionalTrackingHardware TrackingHardware = GetPositionalTrackingHardware(); if (TrackingHardware == ESGPositionalTrackingHardware::Custom) { FSGWristTrackingSettings Settings{GetWristTrackingSettings()}; if (bIsRight) { TrackerLocation += Settings.TrackingHardwareLocationOffsetRightHand; TrackerRotation += Settings.TrackingHardwareRotationOffsetRightHand; } else { TrackerLocation += Settings.TrackingHardwareLocationOffsetLeftHand; TrackerRotation += Settings.TrackingHardwareRotationOffsetLeftHand; } } FVector WristLocation; FRotator WristRotation; USGHapticGlove* Glove = GetGlove(bIsRight); if (!IsValid(Glove)) { return false; } Glove->GetWristLocation(TrackerLocation, TrackerRotation, TrackingHardware, WristLocation, WristRotation); OutWristLocation = MoveTemp(WristLocation); OutWristRotation = FQuat(MoveTemp(WristRotation)); return true; } return false; } bool FSGXRTracker::FImpl::GetWristLocationAndRotation( const EControllerHand DeviceHand, FVector& OutWristLocation, FQuat& OutWristRotation, bool& bOutMotionSourceHandTracking) const { static const FName OpenXRName{TEXT("OpenXR")}; static const FName OpenXRHandTrackingName{TEXT("OpenXRHandTracking")}; const bool bUseMoreSpecificMotionSourceNames = ShouldUseMoreSpecificMotionSourceNames(); static const FName LeftName(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingLeft") : TEXT("Left")); static const FName RightName(bUseMoreSpecificMotionSourceNames ? TEXT("HandTrackingRight") : TEXT("Right")); FName MotionSource; if (DeviceHand == EControllerHand::Left) { MotionSource = LeftName; } else if (DeviceHand == EControllerHand::Right) { MotionSource = RightName; } static constexpr int32 CurrentPlayerControllerIndex = 0; TArray MotionControllers{IModularFeatures::Get().GetModularFeatureImplementations< IMotionController>(IMotionController::GetModularFeatureName())}; for (const IMotionController* MotionController: MotionControllers) { if (!MotionController) { continue; } ETrackingStatus TrackingStatus = MotionController->GetControllerTrackingStatus( CurrentPlayerControllerIndex, MotionSource); if (TrackingStatus != ETrackingStatus::Tracked) { continue; } FName DeviceTypeName{MotionController->GetMotionControllerDeviceTypeName()}; /// NOTE /// We have to always check the wrist tracking data from the OpenXRHandTracking system instead of OpenXR, /// otherwise even if the motion controllers become deactivated their tracking status sometimes mysteriously /// reported as Tracked. if (DeviceTypeName == OpenXRHandTrackingName) { bOutMotionSourceHandTracking = true; return GetOpenXRHandTrackingWristLocationAndRotation(DeviceHand, OutWristLocation, OutWristRotation); } if (DeviceTypeName == OpenXRName) { bOutMotionSourceHandTracking = false; return GetOpenXRWristLocationAndRotation(DeviceHand, MotionController, MotionSource, OutWristLocation, OutWristRotation); } } return false; } bool FSGXRTracker::FImpl::UpdateSGJointData( FSGXRHandState& OutHandState, const EHandKeypoint Joint, const FVector& JointLocation, const FQuat& JointRotation, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{XRTrackingSystem->GetTrackingToWorldTransform()}; const FTransform& WorldToTrackingTransform{TrackingToWorldTransform.Inverse()}; const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); const int32_t JointIndex = static_cast(Joint); FTransform Transform; if (bMotionSourceHandTracking) { XrPosef Pose; Pose.orientation = ToXrQuat(WristRotation * JointRotation); Pose.position = ToXrVector(WristLocation + (WristRotation * JointLocation), WorldToMetersScale); Transform = ToFTransform(MoveTemp(Pose), WorldToMetersScale); Transform *= WorldToTrackingTransform; } else { Transform.SetLocation(WristLocation + (WristRotation * JointLocation)); Transform.SetRotation(WristRotation * JointRotation); } OutHandState.KeypointTransforms[JointIndex] = MoveTemp(Transform); OutHandState.Radii[JointIndex] = 0.01f * WorldToMetersScale; return true; } bool FSGXRTracker::FImpl::UpdateSGWristJointData( FSGXRHandState& OutHandState, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform& TrackingToWorldTransform{XRTrackingSystem->GetTrackingToWorldTransform()}; const FTransform& WorldToTrackingTransform{TrackingToWorldTransform.Inverse()}; const float WorldToMetersScale = XRTrackingSystem->GetWorldToMetersScale(); static constexpr EHandKeypoint Joint = EHandKeypoint::Wrist; static constexpr int32_t JointIndex = static_cast(Joint); FTransform Transform; if (bMotionSourceHandTracking) { XrPosef Pose; Pose.orientation = ToXrQuat(WristRotation); Pose.position = ToXrVector(WristLocation, WorldToMetersScale); Transform = ToFTransform(MoveTemp(Pose), WorldToMetersScale); Transform *= WorldToTrackingTransform; } else { Transform.SetLocation(WristLocation); Transform.SetRotation(WristRotation); } OutHandState.KeypointTransforms[JointIndex] = MoveTemp(Transform); OutHandState.Radii[JointIndex] = 0.01f * WorldToMetersScale; return true; } bool FSGXRTracker::FImpl::UpdateSGPalmJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { if (!ensureAlwaysMsgf(XRTrackingSystem, TEXT("Invalid XR tracking system!"))) { return false; } const FTransform JointTransform{GetMeshBoneRefTransform(DeviceHand, EHandKeypoint::Palm)}; FVector JointLocation{JointTransform.GetLocation()}; FQuat JointRotation{JointTransform.GetRotation()}; if (!UpdateSGJointData(OutHandState, EHandKeypoint::Palm, MoveTemp(JointLocation), MoveTemp(JointRotation), WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGThumbJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { static constexpr int32 ThumbFingerIndex = 0; static constexpr int32 ThumbMetacarpalJointIndex = 0; static constexpr int32 ThumbProximalJointIndex = 1; static constexpr int32 ThumbDistalJointIndex = 2; static constexpr int32 ThumbTipJointIndex = 3; const FVector& ThumbMetacarpalLocation{JointLocations[ThumbFingerIndex][ThumbMetacarpalJointIndex]}; const FVector& ThumbProximalLocation{JointLocations[ThumbFingerIndex][ThumbProximalJointIndex]}; const FVector& ThumbDistalLocation{JointLocations[ThumbFingerIndex][ThumbDistalJointIndex]}; const FVector& ThumbTipLocation{JointLocations[ThumbFingerIndex][ThumbTipJointIndex]}; const FQuat& ThumbMetacarpalRotation{JointRotations[ThumbFingerIndex][ThumbMetacarpalJointIndex]}; const FQuat& ThumbProximalRotation{JointRotations[ThumbFingerIndex][ThumbProximalJointIndex]}; const FQuat& ThumbDistalRotation{JointRotations[ThumbFingerIndex][ThumbDistalJointIndex]}; const FQuat& ThumbTipRotation{JointRotations[ThumbFingerIndex][ThumbTipJointIndex]}; if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbMetacarpal, ThumbMetacarpalLocation, ThumbMetacarpalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbProximal, ThumbProximalLocation, ThumbProximalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbDistal, ThumbDistalLocation, ThumbDistalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::ThumbTip, ThumbTipLocation, ThumbTipRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGIndexJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { static constexpr int32 IndexFingerIndex = 1; static constexpr int32 IndexProximalJointIndex = 0; static constexpr int32 IndexIntermediateJointIndex = 1; static constexpr int32 IndexDistalJointIndex = 2; static constexpr int32 IndexTipJointIndex = 3; const FTransform& IndexMetacarpalTransform{ GetMeshBoneRefTransform(DeviceHand, EHandKeypoint::IndexMetacarpal) }; FVector IndexMetacarpalLocation{IndexMetacarpalTransform.GetLocation()}; FQuat IndexMetacarpalRotation{IndexMetacarpalTransform.GetRotation()}; const FVector& IndexProximalLocation{JointLocations[IndexFingerIndex][IndexProximalJointIndex]}; const FVector& IndexIntermediateLocation{JointLocations[IndexFingerIndex][IndexIntermediateJointIndex]}; const FVector& IndexDistalLocation{JointLocations[IndexFingerIndex][IndexDistalJointIndex]}; const FVector& IndexTipLocation{JointLocations[IndexFingerIndex][IndexTipJointIndex]}; const FQuat& IndexProximalRotation{JointRotations[IndexFingerIndex][IndexProximalJointIndex]}; const FQuat& IndexIntermediateRotation{JointRotations[IndexFingerIndex][IndexIntermediateJointIndex]}; const FQuat& IndexDistalRotation{JointRotations[IndexFingerIndex][IndexDistalJointIndex]}; const FQuat& IndexTipRotation{JointRotations[IndexFingerIndex][IndexTipJointIndex]}; if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexMetacarpal, MoveTemp(IndexMetacarpalLocation), MoveTemp(IndexMetacarpalRotation), WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexProximal, IndexProximalLocation, IndexProximalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexIntermediate, IndexIntermediateLocation, IndexIntermediateRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexDistal, IndexDistalLocation, IndexDistalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::IndexTip, IndexTipLocation, IndexTipRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGMiddleJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { static constexpr int32 MiddleFingerMiddle = 2; static constexpr int32 MiddleProximalJointMiddle = 0; static constexpr int32 MiddleIntermediateJointMiddle = 1; static constexpr int32 MiddleDistalJointMiddle = 2; static constexpr int32 MiddleTipJointMiddle = 3; const FTransform& MiddleMetacarpalTransform{ GetMeshBoneRefTransform(DeviceHand, EHandKeypoint::MiddleMetacarpal) }; FVector MiddleMetacarpalLocation{MiddleMetacarpalTransform.GetLocation()}; FQuat MiddleMetacarpalRotation{MiddleMetacarpalTransform.GetRotation()}; const FVector& MiddleProximalLocation{JointLocations[MiddleFingerMiddle][MiddleProximalJointMiddle]}; const FVector& MiddleIntermediateLocation{JointLocations[MiddleFingerMiddle][MiddleIntermediateJointMiddle]}; const FVector& MiddleDistalLocation{JointLocations[MiddleFingerMiddle][MiddleDistalJointMiddle]}; const FVector& MiddleTipLocation{JointLocations[MiddleFingerMiddle][MiddleTipJointMiddle]}; const FQuat& MiddleProximalRotation{JointRotations[MiddleFingerMiddle][MiddleProximalJointMiddle]}; const FQuat& MiddleIntermediateRotation{JointRotations[MiddleFingerMiddle][MiddleIntermediateJointMiddle]}; const FQuat& MiddleDistalRotation{JointRotations[MiddleFingerMiddle][MiddleDistalJointMiddle]}; const FQuat& MiddleTipRotation{JointRotations[MiddleFingerMiddle][MiddleTipJointMiddle]}; if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleMetacarpal, MoveTemp(MiddleMetacarpalLocation), MoveTemp(MiddleMetacarpalRotation), WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleProximal, MiddleProximalLocation, MiddleProximalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleIntermediate, MiddleIntermediateLocation, MiddleIntermediateRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleDistal, MiddleDistalLocation, MiddleDistalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::MiddleTip, MiddleTipLocation, MiddleTipRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGRingJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { static constexpr int32 RingFingerRing = 3; static constexpr int32 RingProximalJointRing = 0; static constexpr int32 RingIntermediateJointRing = 1; static constexpr int32 RingDistalJointRing = 2; static constexpr int32 RingTipJointRing = 3; const FTransform& RingMetacarpalTransform{ GetMeshBoneRefTransform(DeviceHand, EHandKeypoint::RingMetacarpal) }; FVector RingMetacarpalLocation{RingMetacarpalTransform.GetLocation()}; FQuat RingMetacarpalRotation{RingMetacarpalTransform.GetRotation()}; const FVector& RingProximalLocation{JointLocations[RingFingerRing][RingProximalJointRing]}; const FVector& RingIntermediateLocation{JointLocations[RingFingerRing][RingIntermediateJointRing]}; const FVector& RingDistalLocation{JointLocations[RingFingerRing][RingDistalJointRing]}; const FVector& RingTipLocation{JointLocations[RingFingerRing][RingTipJointRing]}; const FQuat& RingProximalRotation{JointRotations[RingFingerRing][RingProximalJointRing]}; const FQuat& RingIntermediateRotation{JointRotations[RingFingerRing][RingIntermediateJointRing]}; const FQuat& RingDistalRotation{JointRotations[RingFingerRing][RingDistalJointRing]}; const FQuat& RingTipRotation{JointRotations[RingFingerRing][RingTipJointRing]}; if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingMetacarpal, MoveTemp(RingMetacarpalLocation), MoveTemp(RingMetacarpalRotation), WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingProximal, RingProximalLocation, RingProximalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingIntermediate, RingIntermediateLocation, RingIntermediateRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingDistal, RingDistalLocation, RingDistalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::RingTip, RingTipLocation, RingTipRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGLittleJointData( FSGXRHandState& OutHandState, const EControllerHand DeviceHand, const TArray>& JointLocations, const TArray>& JointRotations, const FVector& WristLocation, const FQuat& WristRotation, const bool bMotionSourceHandTracking) const { static constexpr int32 LittleFingerLittle = 4; static constexpr int32 LittleProximalJointLittle = 0; static constexpr int32 LittleIntermediateJointLittle = 1; static constexpr int32 LittleDistalJointLittle = 2; static constexpr int32 LittleTipJointLittle = 3; const FTransform& LittleMetacarpalTransform{ GetMeshBoneRefTransform(DeviceHand, EHandKeypoint::LittleMetacarpal) }; FVector LittleMetacarpalLocation{LittleMetacarpalTransform.GetLocation()}; FQuat LittleMetacarpalRotation{LittleMetacarpalTransform.GetRotation()}; const FVector& LittleProximalLocation{JointLocations[LittleFingerLittle][LittleProximalJointLittle]}; const FVector& LittleIntermediateLocation{JointLocations[LittleFingerLittle][LittleIntermediateJointLittle]}; const FVector& LittleDistalLocation{JointLocations[LittleFingerLittle][LittleDistalJointLittle]}; const FVector& LittleTipLocation{JointLocations[LittleFingerLittle][LittleTipJointLittle]}; const FQuat& LittleProximalRotation{JointRotations[LittleFingerLittle][LittleProximalJointLittle]}; const FQuat& LittleIntermediateRotation{JointRotations[LittleFingerLittle][LittleIntermediateJointLittle]}; const FQuat& LittleDistalRotation{JointRotations[LittleFingerLittle][LittleDistalJointLittle]}; const FQuat& LittleTipRotation{JointRotations[LittleFingerLittle][LittleTipJointLittle]}; if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleMetacarpal, MoveTemp(LittleMetacarpalLocation), MoveTemp(LittleMetacarpalRotation), WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleProximal, LittleProximalLocation, LittleProximalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleIntermediate, LittleIntermediateLocation, LittleIntermediateRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleDistal, LittleDistalLocation, LittleDistalRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGJointData( OutHandState, EHandKeypoint::LittleTip, LittleTipLocation, LittleTipRotation, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } bool FSGXRTracker::FImpl::UpdateSGHandState(const EControllerHand DeviceHand, FSGXRHandState& OutHandState) const { USGHapticGlove* Glove{GetGlove(DeviceHand)}; if (!IsValid(Glove)) { return false; } USGHandPose* HandPose = nullptr; const bool bGotHandPose = Glove->GetHandPose(FImpl::GetWorld(), HandPose); if (!bGotHandPose || !IsValid(HandPose)) { return false; } OutHandState.bReceivedJointPoses = true; const TArray> JointLocations{HandPose->GetJointPositions()}; const TArray> JointRotations{HandPose->GetJointRotationsQ()}; FVector WristLocation; FQuat WristRotation; bool bMotionSourceHandTracking = false; if (!GetWristLocationAndRotation(DeviceHand, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGWristJointData(OutHandState, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGPalmJointData(OutHandState, DeviceHand, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGThumbJointData(OutHandState, DeviceHand, JointLocations, JointRotations, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGIndexJointData(OutHandState, DeviceHand, JointLocations, JointRotations, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGMiddleJointData(OutHandState, DeviceHand, JointLocations, JointRotations, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGRingJointData(OutHandState, DeviceHand, JointLocations, JointRotations, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } if (!UpdateSGLittleJointData(OutHandState, DeviceHand, JointLocations, JointRotations, WristLocation, WristRotation, bMotionSourceHandTracking)) { return false; } return true; } void FSGXRTracker::FImplDeleter::operator()(const FImpl* P) const { delete P; }