migrate away from the deprecated FXRMotionControllerData in favor of FXRMotionControllerState and FXRHandTrackingState on UE 5.5+

2024-10-22 06:55:45 +02:00
parent 9023ecf4f1
commit 3752272cc2
22 changed files with 745 additions and 52 deletions
@@ -998,7 +998,9 @@ FTransform FSGXRTracker::FImpl::GetMeshFingerStartBoneRefTransform(
 }

 bool FSGXRTracker::GetMotionControllerData(
-    UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData)
+    UObject* WorldContextObject,
+    const EControllerHand Hand,
+    FXRMotionControllerData& OutMotionControllerData)
 {
    OutMotionControllerData.DeviceName = NAME_None;
    OutMotionControllerData.ApplicationInstanceID = FApp::GetInstanceId();
@@ -1163,16 +1165,14 @@ bool FSGXRTracker::GetMotionControllerData(
        {
            OutMotionControllerData.DeviceVisualType = EXRVisualType::Hand;

-            /// TODO
-            // This is a just a temporary workaround to silence a UE 5.5 deprecation warning.
-            // Should be refactored with a proper implementation when migrating away from the deprecated
-            // `FXRMotionControllerData` in favor of newly introduced
-            // `FXRMotionControllerState` and `FXRHandTrackingState` structs.
 #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
            bool bTracked = false;
 #endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5 */
+
            OutMotionControllerData.bValid = HandTracker->GetAllKeypointStates(
-                Hand, OutMotionControllerData.HandKeyPositions, OutMotionControllerData.HandKeyRotations,
+                Hand,
+                OutMotionControllerData.HandKeyPositions,
+                OutMotionControllerData.HandKeyRotations,
                OutMotionControllerData.HandKeyRadii
 #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
                , bTracked
@@ -1180,10 +1180,25 @@ bool FSGXRTracker::GetMotionControllerData(
                );

 #if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
-            if (!bTracked)
+            // Unused!
+            (void) bTracked;
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5 */
+
+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
+            // TODO
+            // Begin backward compatibility with the deprecated function, remove this block along with the deprecated
+            // GetAllKeypointStates (though it should be time to remove this entire function as well)!
+            if (!OutMotionControllerData.bValid)
            {
-                return false;
+                PRAGMA_DISABLE_DEPRECATION_WARNINGS
+                OutMotionControllerData.bValid = HandTracker->GetAllKeypointStates(
+                    Hand,
+                    OutMotionControllerData.HandKeyPositions,
+                    OutMotionControllerData.HandKeyRotations,
+                    OutMotionControllerData.HandKeyRadii);
+                PRAGMA_ENABLE_DEPRECATION_WARNINGS
            }
+            // End backward compability.
 #endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5 */

            if (!ensureAlwaysMsgf(
@@ -1204,13 +1219,293 @@ bool FSGXRTracker::GetMotionControllerData(
        OutMotionControllerData.TrackingStatus = ETrackingStatus::Tracked;
    }

-    /// TODO
-    // This is reportedly a WMR-specific convenience function for rapid prototyping. Not sure if it's useful for OpenXR.
-    OutMotionControllerData.bIsGrasped = false;
-
    return OutMotionControllerData.bValid;
 }

+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
+
+bool FSGXRTracker::GetMotionControllerState(
+    UObject* WorldContextObject,
+    const EXRSpaceType XRSpaceType, const EControllerHand Hand, const EXRControllerPoseType XRControllerPoseType,
+    FXRMotionControllerState& OutMotionControllerState)
+{
+    auto ToMotionSourceName = [](
+        const EControllerHand Hand, const EXRControllerPoseType XRControllerPoseType) -> FLazyName
+    {
+        static FLazyName LeftAim = "LeftAim";
+        static FLazyName LeftGrip = "LeftGrip";
+        static FLazyName LeftPalm = "LeftPalm";
+        static FLazyName RightAim = "RightAim";
+        static FLazyName RightGrip = "RightGrip";
+        static FLazyName RightPalm = "RightPalm";
+        if (Hand == EControllerHand::Left)
+        {
+            switch (XRControllerPoseType)
+            {
+            case EXRControllerPoseType::Aim:
+                return LeftAim;
+            case EXRControllerPoseType::Grip:
+                return LeftGrip;
+            case EXRControllerPoseType::Palm:
+                return LeftPalm;
+            default:
+                check(false);
+                return LeftGrip;
+            }
+        }
+        else
+        {
+            switch (XRControllerPoseType)
+            {
+            case EXRControllerPoseType::Aim:
+                return RightAim;
+            case EXRControllerPoseType::Grip:
+                return RightGrip;
+            case EXRControllerPoseType::Palm:
+                return RightPalm;
+            default:
+                check(false);
+                return RightGrip;
+            }
+        }
+    };
+
+    OutMotionControllerState.DeviceName = NAME_None;
+    OutMotionControllerState.ApplicationInstanceID = FApp::GetInstanceId();
+    OutMotionControllerState.TrackingStatus = ETrackingStatus::NotTracked;
+    OutMotionControllerState.Hand = Hand;
+    OutMotionControllerState.XRSpaceType = XRSpaceType;
+    OutMotionControllerState.bValid = false;
+
+    if (!IsValid(WorldContextObject))
+    {
+        return false;
+    }
+
+    if (!IsValid(WorldContextObject))
+    {
+        return false;
+    }
+
+    IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr};
+    if (!XRTrackingSystem)
+    {
+        return false;
+    }
+
+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4
+    IOpenXRHMD* OpenXRHMD{XRTrackingSystem->GetIOpenXRHMD()};
+#else   /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */
+       FOpenXRHMD* OpenXRHMD{static_cast<FOpenXRHMD*>(GEngine->XRSystem.Get())};
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */
+    if (!OpenXRHMD)
+    {
+        return false;
+    }
+
+    TArray<int32> Devices;
+    const int32 HandIndex = static_cast<int32>(Hand);
+    const bool bGotDevices =
+        XRTrackingSystem->EnumerateTrackedDevices(Devices, EXRTrackedDeviceType::Controller);
+    if (bGotDevices && Devices.IsValidIndex(HandIndex))
+    {
+        const XrSession Session{OpenXRHMD->GetSession()};
+        if (Session)
+        {
+            XrInteractionProfileState Profile = {XR_TYPE_INTERACTION_PROFILE_STATE};
+            const XrPath TopLevelUserPath{OpenXRHMD->GetTrackedDevicePath(Devices[HandIndex])};
+            const XrResult CurrentInteractionProfile{
+                xrGetCurrentInteractionProfile(Session, TopLevelUserPath, &Profile)
+            };
+            if (XR_SUCCEEDED(CurrentInteractionProfile) && Profile.interactionProfile != XR_NULL_PATH)
+            {
+                OutMotionControllerState.DeviceName = FOpenXRPath(Profile.interactionProfile);
+            }
+        }
+    }
+
+    if ((Hand == EControllerHand::Left) || (Hand == EControllerHand::Right))
+    {
+        static const FName MotionControllerName("OpenXR");
+        TArray<IMotionController*> MotionControllers{
+            IModularFeatures::Get().GetModularFeatureImplementations<IMotionController>(
+                IMotionController::GetModularFeatureName())
+        };
+        IMotionController* MotionController{nullptr};
+        for (IMotionController* MotionControllerIterator: MotionControllers)
+        {
+            if (!MotionControllerIterator)
+            {
+                continue;
+            }
+
+            const FName DeviceTypeName{MotionControllerIterator->GetMotionControllerDeviceTypeName()};
+            if (DeviceTypeName != MotionControllerName)
+            {
+                continue;
+            }
+
+            MotionController = MotionControllerIterator;
+            break;
+        }
+
+        if (MotionController)
+        {
+            {
+                // Handle the pose that is actually being requested.
+
+                const FName MotionSource{ToMotionSourceName(Hand, XRControllerPoseType)};
+                const FTransform TrackingToWorld{
+                    XRSpaceType == EXRSpaceType::UnrealWorldSpace
+                    ? XRTrackingSystem->GetTrackingToWorldTransform()
+                    : FTransform::Identity
+                };
+                const float WorldToMeters = XRSpaceType == EXRSpaceType::UnrealWorldSpace
+                    ? XRTrackingSystem->GetWorldToMetersScale()
+                    : 100.0f;
+
+                FVector Position{FVector::ZeroVector};
+                FRotator Rotation{FRotator::ZeroRotator};
+
+                bool bGotMotionControllerOrientationAndPosition = MotionController->GetControllerOrientationAndPosition(
+                    0, MotionSource, Rotation, Position, WorldToMeters);
+
+                if (bGotMotionControllerOrientationAndPosition)
+                {
+                    OutMotionControllerState.ControllerLocation = TrackingToWorld.TransformPosition(Position);
+                    OutMotionControllerState.ControllerRotation = TrackingToWorld.TransformRotation(FQuat{Rotation});
+                }
+
+                OutMotionControllerState.bValid |= bGotMotionControllerOrientationAndPosition;
+                OutMotionControllerState.TrackingStatus =
+                    MotionController->GetControllerTrackingStatus(0, MotionSource);
+            }
+
+            {
+                // We always provide the grip transform in Unreal space for XRVisualizationFunctionLibrary;
+                // The bValid and TrackingStatus above are also valid for this pose.
+
+                const FName MotionSource = ToMotionSourceName(Hand, EXRControllerPoseType::Grip);
+                const FTransform TrackingToWorld{XRTrackingSystem->GetTrackingToWorldTransform()};
+                const float WorldToMeters = XRTrackingSystem->GetWorldToMetersScale();
+
+                FVector Position = FVector::ZeroVector;
+                FRotator Rotation = FRotator::ZeroRotator;
+
+                bool bGotMotionControllerOrientationAndPosition = MotionController->GetControllerOrientationAndPosition(
+                    0, MotionSource, Rotation, Position, WorldToMeters);
+
+                if (bGotMotionControllerOrientationAndPosition)
+                {
+                    OutMotionControllerState.GripUnrealSpaceLocation = TrackingToWorld.TransformPosition(Position);
+                    OutMotionControllerState.GripUnrealSpaceRotation =
+                        TrackingToWorld.TransformRotation(FQuat(Rotation));
+                }
+            }
+        }
+    }
+
+    return OutMotionControllerState.bValid;
+}
+
+bool FSGXRTracker::GetHandTrackingState(
+    UObject* WorldContextObject,
+    const EXRSpaceType XRSpaceType, const EControllerHand Hand,
+    FXRHandTrackingState& OutHandTrackingState)
+{
+    OutHandTrackingState.ApplicationInstanceID = FApp::GetInstanceId();
+    OutHandTrackingState.TrackingStatus = ETrackingStatus::NotTracked;
+    OutHandTrackingState.Hand = Hand;
+    OutHandTrackingState.XRSpaceType = XRSpaceType;
+    OutHandTrackingState.bValid = false;
+
+    if (!IsValid(WorldContextObject))
+    {
+        return false;
+    }
+
+    IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr};
+    if (!XRTrackingSystem)
+    {
+        return false;
+    }
+
+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4
+    IOpenXRHMD* OpenXRHMD{XRTrackingSystem->GetIOpenXRHMD()};
+#else   /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */
+    FOpenXRHMD* OpenXRHMD{static_cast<FOpenXRHMD*>(GEngine->XRSystem.Get())};
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 4 */
+    if (!OpenXRHMD)
+    {
+        return false;
+    }
+
+    FName HandTrackerName("OpenXRHandTracking");
+    TArray<IHandTracker*> HandTrackers = IModularFeatures::Get().GetModularFeatureImplementations<IHandTracker>(
+        IHandTracker::GetModularFeatureName());
+    IHandTracker* HandTracker = nullptr;
+    for (auto Itr: HandTrackers)
+    {
+        if (Itr->GetHandTrackerDeviceTypeName() == HandTrackerName)
+        {
+            HandTracker = Itr;
+            break;
+        }
+    }
+
+    if ((Hand == EControllerHand::Left) || (Hand == EControllerHand::Right))
+    {
+        const bool bHandTrackingStateValid = HandTracker && HandTracker->IsHandTrackingStateValid();
+        if (bHandTrackingStateValid)
+        {
+            bool bTracked = false;
+            OutHandTrackingState.bValid = HandTracker->GetAllKeypointStates(
+                Hand,
+                OutHandTrackingState.HandKeyLocations,
+                OutHandTrackingState.HandKeyRotations,
+                OutHandTrackingState.HandKeyRadii,
+                bTracked);
+
+            /// TODO
+            // Begin backward compatibility with the deprecated function, remove this block along with the deprecated
+            // GetAllKeypointStates.
+            if (!OutHandTrackingState.bValid)
+            {
+                PRAGMA_DISABLE_DEPRECATION_WARNINGS
+                OutHandTrackingState.bValid = HandTracker->GetAllKeypointStates(
+                    Hand,
+                    OutHandTrackingState.HandKeyLocations,
+                    OutHandTrackingState.HandKeyRotations,
+                    OutHandTrackingState.HandKeyRadii);
+                PRAGMA_ENABLE_DEPRECATION_WARNINGS
+                bTracked = OutHandTrackingState.bValid;
+                // The inability to distinguish this is the reason for the deprecation.
+            }
+            // End backward compability.
+
+            if (!ensureAlwaysMsgf(
+                !OutHandTrackingState.bValid
+                || (OutHandTrackingState.HandKeyLocations.Num() == EHandKeypointCount
+                    && OutHandTrackingState.HandKeyRotations.Num() == EHandKeypointCount
+                    && OutHandTrackingState.HandKeyRadii.Num() == EHandKeypointCount),
+                TEXT("Corrupted OutHandTrackingState!")))
+            {
+                OutHandTrackingState.bValid = false;
+                return false;
+            }
+
+            if (OutHandTrackingState.bValid)
+            {
+                OutHandTrackingState.TrackingStatus = bTracked ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked;
+            }
+        }
+    }
+
+    return OutHandTrackingState.bValid;
+}
+
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5 */
+
 FSGXRTracker::FSGXRTracker(const TSharedRef<FGenericApplicationMessageHandler>& InMessageHandler)
    : Pimpl(TUniquePtr<FImpl, FImplDeleter>(new FImpl{this}, PimplDeleter))
 {
@@ -1480,7 +1775,7 @@ ETrackingStatus FSGXRTracker::GetControllerTrackingStatus(const int32 Controller
    if (bFallbackToHandTrackingIfNoGloveDetected)
    {
        const FSGXRHandState& HandState{bRight ? GetRightHandState() : GetLeftHandState()};
-        return HandState.bReceivedJointPoses ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked;
+        return HandState.bTracked ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked;
    }

    return ETrackingStatus::NotTracked;
@@ -1606,9 +1901,17 @@ bool FSGXRTracker::GetKeypointState(
    return bGotTransform;
 }

+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
+bool FSGXRTracker::GetAllKeypointStates(
+    const EControllerHand Hand,
+    TArray<FVector>& OutPositions, TArray<FQuat>& OutRotations, TArray<float>& OutRadii,
+    bool& OutIsTracked) const
+#else   /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5 */
 bool FSGXRTracker::GetAllKeypointStates(
    const EControllerHand Hand,
    TArray<FVector>& OutPositions, TArray<FQuat>& OutRotations, TArray<float>& OutRadii) const
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5*/
+
 {
    if (!IsHandTrackingStateValid())
    {
@@ -1629,11 +1932,17 @@ bool FSGXRTracker::GetAllKeypointStates(
        Hand == EControllerHand::Left ? GetLeftHandState() : GetRightHandState()
    };

-    if (!HandState.bReceivedJointPoses)
+    if (!HandState.bHasReceivedJointPoses)
    {
        return false;
    }

+#if ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5
+    OutIsTracked = HandState.bTracked;
+#endif  /* ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 5*/
+
+    // If we ever received poses we return the last data we received, even if not currently tracking to avoid poping
+    // back to (0.0f, 0.0f, 0.0f) and the like.
    OutPositions.Empty(EHandKeypointCount);
    OutRotations.Empty(EHandKeypointCount);
    OutRadii.Empty(EHandKeypointCount);
@@ -1765,8 +2074,9 @@ bool FSGXRTracker::FImpl::UpdateXRHandTrackingData(

    XR_ENSURE(XRLocateHandJointsEXT(OutHandState.HandTracker, &LocateInfo, &OutHandState.Locations));

-    OutHandState.bReceivedJointPoses = OutHandState.Locations.isActive == XR_TRUE;
-    if (!OutHandState.bReceivedJointPoses)
+    OutHandState.bTracked = OutHandState.Locations.isActive == XR_TRUE;
+    OutHandState.bHasReceivedJointPoses |= HandStates->bTracked;
+    if (!OutHandState.bTracked)
    {
        return false;
    }
@@ -1893,21 +2203,16 @@ bool FSGXRTracker::FImpl::GetControllerTransform(
    if (!bTracked && bFallbackToHandTrackingIfNoGloveDetected)
    {
        const FSGXRHandState& HandState{bRight ? Owner->GetRightHandState() : Owner->GetLeftHandState()};
-        if (!HandState.bReceivedJointPoses)
+        if (!HandState.bHasReceivedJointPoses)
        {
            return false;
        }

        const EHandKeypoint KeyPoint = KeyPointInfoPtr->Key;
-        const FTransform& ControllerTransform{
-            bRight
-            ? GetRightHandState().GetTransform(KeyPoint)
-            : GetLeftHandState().GetTransform(KeyPoint)
-        };
-
+        const FTransform& ControllerTransform{HandStates->GetTransform(KeyPoint)};
        OutTransform = ControllerTransform;
        bOutMotionSourceHandTracking = true;
-        bTracked = true;
+        bTracked = HandState.bTracked;
    }

    return bTracked;
@@ -2404,7 +2709,8 @@ bool FSGXRTracker::FImpl::UpdateSGHandState(const bool bRight, FSGXRHandState& O
        return false;
    }

-    OutHandState.bReceivedJointPoses = true;
+    OutHandState.bHasReceivedJointPoses = true;
+    OutHandState.bTracked = true;

    const TArray<TArray<FVector>> JointLocations{HandPose->GetJointPositions()};
    const TArray<TArray<FQuat>> JointRotations{HandPose->GetJointRotationsQ()};
@@ -2469,4 +2775,4 @@ bool FSGXRTracker::FImpl::UpdateSGHandState(const bool bRight, FSGXRHandState& O
 void FSGXRTracker::FImplDeleter::operator()(const FImpl* P) const
 {
    delete P;
-}
+}