implement our own GetMotionControllerData instead of using the one from the XRTrackingSystem as it only picks up the first hand tracking plugin in order to make sure at least internally we use the hand-tracking data from senseglove

2024-08-16 18:00:56 +02:00
parent 54481effbd
commit d0b2ec380e
8 changed files with 354 additions and 20 deletions
@@ -52,6 +52,7 @@
 #include "SGDebug/SGDebugVirtualHand.h"
 #include "SGLog/SGLog.h"
 #include "SGTracking/SGGloveTracker.h"
+#include "SGTracking/SGXRTracker.h"

 struct USGVirtualHandComponent::FImpl
 {
@@ -368,15 +369,14 @@ bool USGVirtualHandComponent::GetMotionControllerData(FXRMotionControllerData& O
 {
    OutMotionControllerData.bValid = false;

-    IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr};
-    if (!XRTrackingSystem)
+    const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left;
+    const bool bGotMotionControllerData =
+        FSGXRTracker::GetMotionControllerData(this, Hand, OutMotionControllerData);
+    if (!bGotMotionControllerData || !OutMotionControllerData.bValid)
    {
        return false;
    }

-    const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left;
-    XRTrackingSystem->GetMotionControllerData(this, Hand, OutMotionControllerData);
-
    return OutMotionControllerData.bValid;
 }

@@ -610,18 +610,50 @@ void USGVirtualHandComponent::FImpl::SetVisibility(const bool bInVisible)

 void USGVirtualHandComponent::FImpl::UpdateVisibility()
 {
-    const bool bUpdateVisibility = !Owner->IsVisibleWhenHandDataUnavailable();
-    if (!bUpdateVisibility)
-    {
-        return;
-    }
-
    FXRMotionControllerData MotionControllerData;
    const bool bGotMotionControllerData = Owner->GetMotionControllerData(MotionControllerData);
    const bool bHandDataAvailable = bGotMotionControllerData && MotionControllerData.bValid;
-    const bool bVisibleHand = Owner->GetVisibleFlag();
-    const bool bSetNewVisibility =
-        (bHandDataAvailable && !bVisibleHand) || (!bHandDataAvailable && bVisibleHand);
+    const bool bIsHandVisible = Owner->GetVisibleFlag();
+    const bool bShouldHandBeVisibleWhenHandDataUnavailable = Owner->IsVisibleWhenHandDataUnavailable();
+
+    bool bSetNewVisibility = false;
+
+    if (bHandDataAvailable)
+    {
+        if (bIsHandVisible)
+        {
+            bSetNewVisibility = false;
+        }
+        else
+        {
+            bSetNewVisibility = true;
+        }
+    }
+    else
+    {
+        if (bIsHandVisible)
+        {
+            if (bShouldHandBeVisibleWhenHandDataUnavailable)
+            {
+                bSetNewVisibility = false;
+            }
+            else
+            {
+                bSetNewVisibility = true;
+            }
+        }
+        else
+        {
+            if (bShouldHandBeVisibleWhenHandDataUnavailable)
+            {
+                bSetNewVisibility = true;
+            }
+            else
+            {
+                bSetNewVisibility = false;
+            }
+        }
+    }

    if (bSetNewVisibility)
    {
@@ -639,7 +671,7 @@ void USGVirtualHandComponent::FImpl::DrawDebugVirtualHand() const
    }

    FXRMotionControllerData MotionControllerData;
-    const bool bGotMotionControllerData{Owner->GetMotionControllerData(MotionControllerData)};
+    const bool bGotMotionControllerData = Owner->GetMotionControllerData(MotionControllerData);
    if (!bGotMotionControllerData || !MotionControllerData.bValid)
    {
        return;
@@ -39,6 +39,7 @@

 #include "SGDebug/SGDebugGizmo.h"
 #include "SGSettings/SGSettings.h"
+#include "SGTracking/SGXRTracker.h"

 struct USGWristTrackerComponent::FImpl
 {
@@ -184,15 +185,14 @@ bool USGWristTrackerComponent::GetMotionControllerData(FXRMotionControllerData&
 {
    OutMotionControllerData.bValid = false;

-    IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr};
-    if (!XRTrackingSystem)
+    const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left;
+    const bool bGotMotionControllerData =
+        FSGXRTracker::GetMotionControllerData(this, Hand, OutMotionControllerData);
+    if (!bGotMotionControllerData || !OutMotionControllerData.bValid)
    {
        return false;
    }

-    const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left;
-    XRTrackingSystem->GetMotionControllerData(this, Hand, OutMotionControllerData);
-
    return OutMotionControllerData.bValid;
 }

@@ -52,6 +52,8 @@ public class SenseGlove : ModuleRules
                "EngineSettings",
                "HeadMountedDisplay",
                "InputCore",
+                "InputDevice",
+                "OpenXRHMD",
                // NOTE
                // UE 5.3+ provides UXRDeviceVisualizationComponent as part of the XRBase plugin.
                // UE 5.2 provides UXRDeviceVisualizationComponent as part of the HeadMountDisplay module.
@@ -41,6 +41,7 @@
 #include "Engine/SkeletalMesh.h"
 #include "Engine/World.h"
 #include "GameFramework/WorldSettings.h"
+#include "IOpenXRHMD.h"
 #include "IOpenXRHMDModule.h"
 #include "IXRTrackingSystem.h"
 #include "OpenXRCore.h"
@@ -972,6 +973,192 @@ FTransform FSGXRTracker::FImpl::GetMeshFingerStartBoneRefTransform(
    return Transform;
 }

+bool FSGXRTracker::GetMotionControllerData(
+    UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData)
+{
+    OutMotionControllerData.DeviceName = NAME_None;
+    OutMotionControllerData.ApplicationInstanceID = FApp::GetInstanceId();
+    OutMotionControllerData.DeviceVisualType = EXRVisualType::Controller;
+    OutMotionControllerData.TrackingStatus = ETrackingStatus::NotTracked;
+    OutMotionControllerData.HandIndex = Hand;
+    OutMotionControllerData.bValid = false;
+
+    if (!IsValid(WorldContextObject))
+    {
+        return false;
+    }
+
+    IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr};
+    if (!XRTrackingSystem)
+    {
+        return false;
+    }
+
+    IOpenXRHMD* OpenXRHMD{XRTrackingSystem->GetIOpenXRHMD()};
+    if (!OpenXRHMD)
+    {
+        return false;
+    }
+
+    TArray<int32> Devices;
+    const int32 HandIndex = static_cast<int32>(Hand);
+    const bool bGotDevices =
+        XRTrackingSystem->EnumerateTrackedDevices(Devices, EXRTrackedDeviceType::Controller);
+    if (bGotDevices && Devices.IsValidIndex(HandIndex))
+    {
+        const XrSession Session{OpenXRHMD->GetSession()};
+        if (Session)
+        {
+            XrInteractionProfileState Profile = {XR_TYPE_INTERACTION_PROFILE_STATE};
+            const XrPath TopLevelUserPath{OpenXRHMD->GetTrackedDevicePath(Devices[HandIndex])};
+            const XrResult CurrentInteractionProfile{
+                xrGetCurrentInteractionProfile(Session, TopLevelUserPath, &Profile)
+            };
+            if (XR_SUCCEEDED(CurrentInteractionProfile) && Profile.interactionProfile != XR_NULL_PATH)
+            {
+                OutMotionControllerData.DeviceName = FOpenXRPath(Profile.interactionProfile);
+            }
+        }
+    }
+
+    static const FName HandTrackerName{"OpenXRHandTracking"};
+    TArray<IHandTracker*> HandTrackers = IModularFeatures::Get().GetModularFeatureImplementations<IHandTracker>(
+        IHandTracker::GetModularFeatureName());
+    IHandTracker* HandTracker{nullptr};
+    for (IHandTracker* HandTrackerIterator: HandTrackers)
+    {
+        if (!HandTrackerIterator)
+        {
+            continue;
+        }
+
+        const FName DeviceTypeName{HandTrackerIterator->GetHandTrackerDeviceTypeName()};
+        if (DeviceTypeName != HandTrackerName)
+        {
+            continue;
+        }
+
+        FSGXRTracker* SGXRTracker{static_cast<FSGXRTracker*>(HandTrackerIterator)};
+        if (!SGXRTracker)
+        {
+            continue;
+        }
+
+        HandTracker = SGXRTracker;
+        break;
+    }
+
+    if (Hand == EControllerHand::Left || Hand == EControllerHand::Right)
+    {
+        static const FName MotionControllerName("OpenXR");
+        TArray<IMotionController*> MotionControllers{
+            IModularFeatures::Get().GetModularFeatureImplementations<IMotionController>(
+                IMotionController::GetModularFeatureName())
+        };
+        IMotionController* MotionController{nullptr};
+        for (IMotionController* MotionControllerIterator: MotionControllers)
+        {
+            if (!MotionControllerIterator)
+            {
+                continue;
+            }
+
+            const FName DeviceTypeName{MotionControllerIterator->GetMotionControllerDeviceTypeName()};
+            if (DeviceTypeName != MotionControllerName)
+            {
+                continue;
+            }
+
+            MotionController = MotionControllerIterator;
+            break;
+        }
+
+        const float WorldToMeters = XRTrackingSystem->GetWorldToMetersScale();
+        if (MotionController)
+        {
+            const FTransform TrackingToWorld{XRTrackingSystem->GetTrackingToWorldTransform()};
+
+            {
+                const FName AimSource{Hand == EControllerHand::Left ? FName{TEXT("LeftAim")} : FName{TEXT("RightAim")}};
+                FRotator AimSourceRotation;
+                FVector AimSourcePosition;
+                const bool bGotAimOrientationAndPosition =
+                    MotionController->GetControllerOrientationAndPosition(
+                        0, AimSource, AimSourceRotation, AimSourcePosition, WorldToMeters);
+                if (bGotAimOrientationAndPosition)
+                {
+                    OutMotionControllerData.AimPosition = TrackingToWorld.TransformPosition(AimSourcePosition);
+                    OutMotionControllerData.AimRotation =
+                        TrackingToWorld.TransformRotation(AimSourceRotation.Quaternion());
+                }
+                OutMotionControllerData.bValid |= bGotAimOrientationAndPosition;
+            }
+
+            const FName GripSource{Hand == EControllerHand::Left ? FName{TEXT("LeftGrip")} : FName{TEXT("RightGrip")}};
+            {
+                FRotator GripRotation;
+                FVector GripPosition;
+                const bool bGotGripSourceOrientationAndPosition =
+                    MotionController->GetControllerOrientationAndPosition(
+                        0, GripSource, GripRotation, GripPosition, WorldToMeters);
+                if (bGotGripSourceOrientationAndPosition)
+                {
+                    OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(GripPosition);
+                    OutMotionControllerData.GripRotation =
+                        TrackingToWorld.TransformRotation(GripRotation.Quaternion());
+                }
+                OutMotionControllerData.bValid |= bGotGripSourceOrientationAndPosition;
+            }
+
+            {
+                const FName PalmSource{
+                    Hand == EControllerHand::Left ? FName{TEXT("LeftPalm")} : FName{TEXT("RightPalm")}
+                };
+                FRotator PalmRotation;
+                FVector PalmPosition;
+                const bool bGotPalmOrientationAndPosition = MotionController->GetControllerOrientationAndPosition(
+                    0, PalmSource, PalmRotation, PalmPosition, WorldToMeters);
+                if (bGotPalmOrientationAndPosition)
+                {
+                    OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(PalmPosition);
+                    OutMotionControllerData.GripRotation =
+                        TrackingToWorld.TransformRotation(PalmRotation.Quaternion());
+                }
+                OutMotionControllerData.bValid |= bGotPalmOrientationAndPosition;
+            }
+
+            OutMotionControllerData.TrackingStatus = MotionController->GetControllerTrackingStatus(0, GripSource);
+        }
+
+        const bool bHandTrackingStateValid = HandTracker && HandTracker->IsHandTrackingStateValid();
+        if (bHandTrackingStateValid)
+        {
+            OutMotionControllerData.DeviceVisualType = EXRVisualType::Hand;
+
+            OutMotionControllerData.bValid = HandTracker->GetAllKeypointStates(
+                Hand, OutMotionControllerData.HandKeyPositions, OutMotionControllerData.HandKeyRotations,
+                OutMotionControllerData.HandKeyRadii);
+
+            if (!ensureAlwaysMsgf(
+                !OutMotionControllerData.bValid
+                || (OutMotionControllerData.HandKeyPositions.Num() == EHandKeypointCount
+                    && OutMotionControllerData.HandKeyRotations.Num() == EHandKeypointCount
+                    && OutMotionControllerData.HandKeyRadii.Num() == EHandKeypointCount),
+                TEXT("Corrupted OutMotionControllerData!")))
+            {
+                OutMotionControllerData.bValid = false;
+                return false;
+            }
+        }
+    }
+
+    /// TODO
+    // This is reportedly a WMR-specific convenience function for rapid prototyping. Not sure if it's useful for OpenXR.
+    OutMotionControllerData.bIsGrasped = false;
+
+    return OutMotionControllerData.bValid;
+}
+
 FSGXRTracker::FSGXRTracker(const TSharedRef<FGenericApplicationMessageHandler>& InMessageHandler)
    : Pimpl(TUniquePtr<FImpl, FImplDeleter>(new FImpl{this}, PimplDeleter))
 {
@@ -54,6 +54,10 @@ class SENSEGLOVETRACKING_API FSGXRTracker final : public IOpenXRExtensionPlugin,
                                                  public FXRMotionControllerBase,
                                                  public IHandTracker
 {
+public:
+    static bool GetMotionControllerData(UObject* WorldContextObject, EControllerHand Hand,
+                                        FXRMotionControllerData& OutMotionControllerData);
+
 private:
    struct FImpl;

@@ -0,0 +1,46 @@
+/**
+ * @file
+ *
+ * @author  Mamadou Babaei <mamadou@senseglove.com>
+ *
+ * @section LICENSE
+ *
+ * (The MIT License)
+ *
+ * Copyright (c) 2020 - 2024 SenseGlove
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ *
+ */
+
+
+#include "SGTrackingKismet/SGXRTrackerKismetLibrary.h"
+
+#include "UObject/Object.h"
+
+#include "SGTracking/SGXRTracker.h"
+
+bool USGXRTrackerKismetLibrary::GetMotionControllerData(
+    UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData)
+{
+    return FSGXRTracker::GetMotionControllerData(WorldContextObject, Hand, OutMotionControllerData);
+}
@@ -0,0 +1,58 @@
+/**
+ * @file
+ *
+ * @author  Mamadou Babaei <mamadou@senseglove.com>
+ *
+ * @section LICENSE
+ *
+ * (The MIT License)
+ *
+ * Copyright (c) 2020 - 2024 SenseGlove
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ *
+ */
+
+
+#pragma once
+
+#include "HeadMountedDisplayTypes.h"
+#include "InputCoreTypes.h"
+#include "UObject/ObjectMacros.h"
+
+#include "SGKismet/SGBlueprintFunctionLibrary.h"
+
+#include "SGXRTrackerKismetLibrary.generated.h"
+
+class UObject;
+
+UCLASS(meta=(ScriptName = "SenseGloveHeadMountDisplayKismetLibrary"))
+class SENSEGLOVETRACKINGKISMET_API USGXRTrackerKismetLibrary final : public USGBlueprintFunctionLibrary
+{
+    GENERATED_BODY()
+
+public:
+    UFUNCTION(BlueprintCallable, Category="SenseGlove | Tracking | XR Tracker",
+        meta=(WorldContext="WorldContextObject"))
+    static bool GetMotionControllerData(UObject* WorldContextObject, EControllerHand Hand,
+                                        FXRMotionControllerData& OutMotionControllerData);
+};
@@ -48,6 +48,11 @@ public class SenseGloveTrackingKismet : ModuleRules
                "Core",
                "CoreUObject",
                "Engine",
+                "HeadMountedDisplay",
+                "InputCore",
+                "InputDevice",
+                "OpenXRHMD",
+                "SenseGloveBuildHacks",
                "SenseGloveCore",
                "SenseGloveLog",
                "SenseGloveKismet",