From d0b2ec380ef24a17216395b73d2a18d1bf282069 Mon Sep 17 00:00:00 2001 From: Mamadou Babaei Date: Fri, 5 Jul 2024 22:08:21 +0200 Subject: [PATCH] implement our own GetMotionControllerData instead of using the one from the XRTrackingSystem as it only picks up the first hand tracking plugin in order to make sure at least internally we use the hand-tracking data from senseglove --- .../Components/SGVirtualHandComponent.cpp | 62 ++++-- .../Components/SGWristTrackerComponent.cpp | 10 +- Source/SenseGlove/SenseGlove.Build.cs | 2 + .../Private/SGTracking/SGXRTracker.cpp | 187 ++++++++++++++++++ .../Public/SGTracking/SGXRTracker.h | 4 + .../SGXRTrackerKismetLibrary.cpp | 46 +++++ .../SGXRTrackerKismetLibrary.h | 58 ++++++ .../SenseGloveTrackingKismet.Build.cs | 5 + 8 files changed, 354 insertions(+), 20 deletions(-) create mode 100644 Source/SenseGloveTrackingKismet/Private/SGTrackingKismet/SGXRTrackerKismetLibrary.cpp create mode 100644 Source/SenseGloveTrackingKismet/Public/SGTrackingKismet/SGXRTrackerKismetLibrary.h diff --git a/Source/SenseGlove/Private/SenseGlove/Components/SGVirtualHandComponent.cpp b/Source/SenseGlove/Private/SenseGlove/Components/SGVirtualHandComponent.cpp index 450d049b..3fe606b1 100644 --- a/Source/SenseGlove/Private/SenseGlove/Components/SGVirtualHandComponent.cpp +++ b/Source/SenseGlove/Private/SenseGlove/Components/SGVirtualHandComponent.cpp @@ -52,6 +52,7 @@ #include "SGDebug/SGDebugVirtualHand.h" #include "SGLog/SGLog.h" #include "SGTracking/SGGloveTracker.h" +#include "SGTracking/SGXRTracker.h" struct USGVirtualHandComponent::FImpl { @@ -368,15 +369,14 @@ bool USGVirtualHandComponent::GetMotionControllerData(FXRMotionControllerData& O { OutMotionControllerData.bValid = false; - IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr}; - if (!XRTrackingSystem) + const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left; + const bool bGotMotionControllerData = + FSGXRTracker::GetMotionControllerData(this, Hand, OutMotionControllerData); + if (!bGotMotionControllerData || !OutMotionControllerData.bValid) { return false; } - const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left; - XRTrackingSystem->GetMotionControllerData(this, Hand, OutMotionControllerData); - return OutMotionControllerData.bValid; } @@ -610,18 +610,50 @@ void USGVirtualHandComponent::FImpl::SetVisibility(const bool bInVisible) void USGVirtualHandComponent::FImpl::UpdateVisibility() { - const bool bUpdateVisibility = !Owner->IsVisibleWhenHandDataUnavailable(); - if (!bUpdateVisibility) - { - return; - } - FXRMotionControllerData MotionControllerData; const bool bGotMotionControllerData = Owner->GetMotionControllerData(MotionControllerData); const bool bHandDataAvailable = bGotMotionControllerData && MotionControllerData.bValid; - const bool bVisibleHand = Owner->GetVisibleFlag(); - const bool bSetNewVisibility = - (bHandDataAvailable && !bVisibleHand) || (!bHandDataAvailable && bVisibleHand); + const bool bIsHandVisible = Owner->GetVisibleFlag(); + const bool bShouldHandBeVisibleWhenHandDataUnavailable = Owner->IsVisibleWhenHandDataUnavailable(); + + bool bSetNewVisibility = false; + + if (bHandDataAvailable) + { + if (bIsHandVisible) + { + bSetNewVisibility = false; + } + else + { + bSetNewVisibility = true; + } + } + else + { + if (bIsHandVisible) + { + if (bShouldHandBeVisibleWhenHandDataUnavailable) + { + bSetNewVisibility = false; + } + else + { + bSetNewVisibility = true; + } + } + else + { + if (bShouldHandBeVisibleWhenHandDataUnavailable) + { + bSetNewVisibility = true; + } + else + { + bSetNewVisibility = false; + } + } + } if (bSetNewVisibility) { @@ -639,7 +671,7 @@ void USGVirtualHandComponent::FImpl::DrawDebugVirtualHand() const } FXRMotionControllerData MotionControllerData; - const bool bGotMotionControllerData{Owner->GetMotionControllerData(MotionControllerData)}; + const bool bGotMotionControllerData = Owner->GetMotionControllerData(MotionControllerData); if (!bGotMotionControllerData || !MotionControllerData.bValid) { return; diff --git a/Source/SenseGlove/Private/SenseGlove/Components/SGWristTrackerComponent.cpp b/Source/SenseGlove/Private/SenseGlove/Components/SGWristTrackerComponent.cpp index 2a125179..e3e87a62 100644 --- a/Source/SenseGlove/Private/SenseGlove/Components/SGWristTrackerComponent.cpp +++ b/Source/SenseGlove/Private/SenseGlove/Components/SGWristTrackerComponent.cpp @@ -39,6 +39,7 @@ #include "SGDebug/SGDebugGizmo.h" #include "SGSettings/SGSettings.h" +#include "SGTracking/SGXRTracker.h" struct USGWristTrackerComponent::FImpl { @@ -184,15 +185,14 @@ bool USGWristTrackerComponent::GetMotionControllerData(FXRMotionControllerData& { OutMotionControllerData.bValid = false; - IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr}; - if (!XRTrackingSystem) + const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left; + const bool bGotMotionControllerData = + FSGXRTracker::GetMotionControllerData(this, Hand, OutMotionControllerData); + if (!bGotMotionControllerData || !OutMotionControllerData.bValid) { return false; } - const EControllerHand Hand = IsRight() ? EControllerHand::Right : EControllerHand::Left; - XRTrackingSystem->GetMotionControllerData(this, Hand, OutMotionControllerData); - return OutMotionControllerData.bValid; } diff --git a/Source/SenseGlove/SenseGlove.Build.cs b/Source/SenseGlove/SenseGlove.Build.cs index 85c6da39..b64cce26 100644 --- a/Source/SenseGlove/SenseGlove.Build.cs +++ b/Source/SenseGlove/SenseGlove.Build.cs @@ -52,6 +52,8 @@ public class SenseGlove : ModuleRules "EngineSettings", "HeadMountedDisplay", "InputCore", + "InputDevice", + "OpenXRHMD", // NOTE // UE 5.3+ provides UXRDeviceVisualizationComponent as part of the XRBase plugin. // UE 5.2 provides UXRDeviceVisualizationComponent as part of the HeadMountDisplay module. diff --git a/Source/SenseGloveTracking/Private/SGTracking/SGXRTracker.cpp b/Source/SenseGloveTracking/Private/SGTracking/SGXRTracker.cpp index cd6ddc9a..30c436c0 100644 --- a/Source/SenseGloveTracking/Private/SGTracking/SGXRTracker.cpp +++ b/Source/SenseGloveTracking/Private/SGTracking/SGXRTracker.cpp @@ -41,6 +41,7 @@ #include "Engine/SkeletalMesh.h" #include "Engine/World.h" #include "GameFramework/WorldSettings.h" +#include "IOpenXRHMD.h" #include "IOpenXRHMDModule.h" #include "IXRTrackingSystem.h" #include "OpenXRCore.h" @@ -972,6 +973,192 @@ FTransform FSGXRTracker::FImpl::GetMeshFingerStartBoneRefTransform( return Transform; } +bool FSGXRTracker::GetMotionControllerData( + UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData) +{ + OutMotionControllerData.DeviceName = NAME_None; + OutMotionControllerData.ApplicationInstanceID = FApp::GetInstanceId(); + OutMotionControllerData.DeviceVisualType = EXRVisualType::Controller; + OutMotionControllerData.TrackingStatus = ETrackingStatus::NotTracked; + OutMotionControllerData.HandIndex = Hand; + OutMotionControllerData.bValid = false; + + if (!IsValid(WorldContextObject)) + { + return false; + } + + IXRTrackingSystem* XRTrackingSystem{GEngine ? GEngine->XRSystem.Get() : nullptr}; + if (!XRTrackingSystem) + { + return false; + } + + IOpenXRHMD* OpenXRHMD{XRTrackingSystem->GetIOpenXRHMD()}; + if (!OpenXRHMD) + { + return false; + } + + TArray Devices; + const int32 HandIndex = static_cast(Hand); + const bool bGotDevices = + XRTrackingSystem->EnumerateTrackedDevices(Devices, EXRTrackedDeviceType::Controller); + if (bGotDevices && Devices.IsValidIndex(HandIndex)) + { + const XrSession Session{OpenXRHMD->GetSession()}; + if (Session) + { + XrInteractionProfileState Profile = {XR_TYPE_INTERACTION_PROFILE_STATE}; + const XrPath TopLevelUserPath{OpenXRHMD->GetTrackedDevicePath(Devices[HandIndex])}; + const XrResult CurrentInteractionProfile{ + xrGetCurrentInteractionProfile(Session, TopLevelUserPath, &Profile) + }; + if (XR_SUCCEEDED(CurrentInteractionProfile) && Profile.interactionProfile != XR_NULL_PATH) + { + OutMotionControllerData.DeviceName = FOpenXRPath(Profile.interactionProfile); + } + } + } + + static const FName HandTrackerName{"OpenXRHandTracking"}; + TArray HandTrackers = IModularFeatures::Get().GetModularFeatureImplementations( + IHandTracker::GetModularFeatureName()); + IHandTracker* HandTracker{nullptr}; + for (IHandTracker* HandTrackerIterator: HandTrackers) + { + if (!HandTrackerIterator) + { + continue; + } + + const FName DeviceTypeName{HandTrackerIterator->GetHandTrackerDeviceTypeName()}; + if (DeviceTypeName != HandTrackerName) + { + continue; + } + + FSGXRTracker* SGXRTracker{static_cast(HandTrackerIterator)}; + if (!SGXRTracker) + { + continue; + } + + HandTracker = SGXRTracker; + break; + } + + if (Hand == EControllerHand::Left || Hand == EControllerHand::Right) + { + static const FName MotionControllerName("OpenXR"); + TArray MotionControllers{ + IModularFeatures::Get().GetModularFeatureImplementations( + IMotionController::GetModularFeatureName()) + }; + IMotionController* MotionController{nullptr}; + for (IMotionController* MotionControllerIterator: MotionControllers) + { + if (!MotionControllerIterator) + { + continue; + } + + const FName DeviceTypeName{MotionControllerIterator->GetMotionControllerDeviceTypeName()}; + if (DeviceTypeName != MotionControllerName) + { + continue; + } + + MotionController = MotionControllerIterator; + break; + } + + const float WorldToMeters = XRTrackingSystem->GetWorldToMetersScale(); + if (MotionController) + { + const FTransform TrackingToWorld{XRTrackingSystem->GetTrackingToWorldTransform()}; + + { + const FName AimSource{Hand == EControllerHand::Left ? FName{TEXT("LeftAim")} : FName{TEXT("RightAim")}}; + FRotator AimSourceRotation; + FVector AimSourcePosition; + const bool bGotAimOrientationAndPosition = + MotionController->GetControllerOrientationAndPosition( + 0, AimSource, AimSourceRotation, AimSourcePosition, WorldToMeters); + if (bGotAimOrientationAndPosition) + { + OutMotionControllerData.AimPosition = TrackingToWorld.TransformPosition(AimSourcePosition); + OutMotionControllerData.AimRotation = + TrackingToWorld.TransformRotation(AimSourceRotation.Quaternion()); + } + OutMotionControllerData.bValid |= bGotAimOrientationAndPosition; + } + + const FName GripSource{Hand == EControllerHand::Left ? FName{TEXT("LeftGrip")} : FName{TEXT("RightGrip")}}; + { + FRotator GripRotation; + FVector GripPosition; + const bool bGotGripSourceOrientationAndPosition = + MotionController->GetControllerOrientationAndPosition( + 0, GripSource, GripRotation, GripPosition, WorldToMeters); + if (bGotGripSourceOrientationAndPosition) + { + OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(GripPosition); + OutMotionControllerData.GripRotation = + TrackingToWorld.TransformRotation(GripRotation.Quaternion()); + } + OutMotionControllerData.bValid |= bGotGripSourceOrientationAndPosition; + } + + { + const FName PalmSource{ + Hand == EControllerHand::Left ? FName{TEXT("LeftPalm")} : FName{TEXT("RightPalm")} + }; + FRotator PalmRotation; + FVector PalmPosition; + const bool bGotPalmOrientationAndPosition = MotionController->GetControllerOrientationAndPosition( + 0, PalmSource, PalmRotation, PalmPosition, WorldToMeters); + if (bGotPalmOrientationAndPosition) + { + OutMotionControllerData.GripPosition = TrackingToWorld.TransformPosition(PalmPosition); + OutMotionControllerData.GripRotation = + TrackingToWorld.TransformRotation(PalmRotation.Quaternion()); + } + OutMotionControllerData.bValid |= bGotPalmOrientationAndPosition; + } + + OutMotionControllerData.TrackingStatus = MotionController->GetControllerTrackingStatus(0, GripSource); + } + + const bool bHandTrackingStateValid = HandTracker && HandTracker->IsHandTrackingStateValid(); + if (bHandTrackingStateValid) + { + OutMotionControllerData.DeviceVisualType = EXRVisualType::Hand; + + OutMotionControllerData.bValid = HandTracker->GetAllKeypointStates( + Hand, OutMotionControllerData.HandKeyPositions, OutMotionControllerData.HandKeyRotations, + OutMotionControllerData.HandKeyRadii); + + if (!ensureAlwaysMsgf( + !OutMotionControllerData.bValid + || (OutMotionControllerData.HandKeyPositions.Num() == EHandKeypointCount + && OutMotionControllerData.HandKeyRotations.Num() == EHandKeypointCount + && OutMotionControllerData.HandKeyRadii.Num() == EHandKeypointCount), + TEXT("Corrupted OutMotionControllerData!"))) + { + OutMotionControllerData.bValid = false; + return false; + } + } + } + + /// TODO + // This is reportedly a WMR-specific convenience function for rapid prototyping. Not sure if it's useful for OpenXR. + OutMotionControllerData.bIsGrasped = false; + + return OutMotionControllerData.bValid; +} + FSGXRTracker::FSGXRTracker(const TSharedRef& InMessageHandler) : Pimpl(TUniquePtr(new FImpl{this}, PimplDeleter)) { diff --git a/Source/SenseGloveTracking/Public/SGTracking/SGXRTracker.h b/Source/SenseGloveTracking/Public/SGTracking/SGXRTracker.h index a7bcfb05..2cf9319c 100644 --- a/Source/SenseGloveTracking/Public/SGTracking/SGXRTracker.h +++ b/Source/SenseGloveTracking/Public/SGTracking/SGXRTracker.h @@ -54,6 +54,10 @@ class SENSEGLOVETRACKING_API FSGXRTracker final : public IOpenXRExtensionPlugin, public FXRMotionControllerBase, public IHandTracker { +public: + static bool GetMotionControllerData(UObject* WorldContextObject, EControllerHand Hand, + FXRMotionControllerData& OutMotionControllerData); + private: struct FImpl; diff --git a/Source/SenseGloveTrackingKismet/Private/SGTrackingKismet/SGXRTrackerKismetLibrary.cpp b/Source/SenseGloveTrackingKismet/Private/SGTrackingKismet/SGXRTrackerKismetLibrary.cpp new file mode 100644 index 00000000..138db9fe --- /dev/null +++ b/Source/SenseGloveTrackingKismet/Private/SGTrackingKismet/SGXRTrackerKismetLibrary.cpp @@ -0,0 +1,46 @@ +/** + * @file + * + * @author Mamadou Babaei + * + * @section LICENSE + * + * (The MIT License) + * + * Copyright (c) 2020 - 2024 SenseGlove + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * @section DESCRIPTION + * + * + */ + + +#include "SGTrackingKismet/SGXRTrackerKismetLibrary.h" + +#include "UObject/Object.h" + +#include "SGTracking/SGXRTracker.h" + +bool USGXRTrackerKismetLibrary::GetMotionControllerData( + UObject* WorldContextObject, const EControllerHand Hand, FXRMotionControllerData& OutMotionControllerData) +{ + return FSGXRTracker::GetMotionControllerData(WorldContextObject, Hand, OutMotionControllerData); +} \ No newline at end of file diff --git a/Source/SenseGloveTrackingKismet/Public/SGTrackingKismet/SGXRTrackerKismetLibrary.h b/Source/SenseGloveTrackingKismet/Public/SGTrackingKismet/SGXRTrackerKismetLibrary.h new file mode 100644 index 00000000..ee133692 --- /dev/null +++ b/Source/SenseGloveTrackingKismet/Public/SGTrackingKismet/SGXRTrackerKismetLibrary.h @@ -0,0 +1,58 @@ +/** + * @file + * + * @author Mamadou Babaei + * + * @section LICENSE + * + * (The MIT License) + * + * Copyright (c) 2020 - 2024 SenseGlove + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * @section DESCRIPTION + * + * + */ + + +#pragma once + +#include "HeadMountedDisplayTypes.h" +#include "InputCoreTypes.h" +#include "UObject/ObjectMacros.h" + +#include "SGKismet/SGBlueprintFunctionLibrary.h" + +#include "SGXRTrackerKismetLibrary.generated.h" + +class UObject; + +UCLASS(meta=(ScriptName = "SenseGloveHeadMountDisplayKismetLibrary")) +class SENSEGLOVETRACKINGKISMET_API USGXRTrackerKismetLibrary final : public USGBlueprintFunctionLibrary +{ + GENERATED_BODY() + +public: + UFUNCTION(BlueprintCallable, Category="SenseGlove | Tracking | XR Tracker", + meta=(WorldContext="WorldContextObject")) + static bool GetMotionControllerData(UObject* WorldContextObject, EControllerHand Hand, + FXRMotionControllerData& OutMotionControllerData); +}; diff --git a/Source/SenseGloveTrackingKismet/SenseGloveTrackingKismet.Build.cs b/Source/SenseGloveTrackingKismet/SenseGloveTrackingKismet.Build.cs index c8b1e8f1..0053816b 100644 --- a/Source/SenseGloveTrackingKismet/SenseGloveTrackingKismet.Build.cs +++ b/Source/SenseGloveTrackingKismet/SenseGloveTrackingKismet.Build.cs @@ -48,6 +48,11 @@ public class SenseGloveTrackingKismet : ModuleRules "Core", "CoreUObject", "Engine", + "HeadMountedDisplay", + "InputCore", + "InputDevice", + "OpenXRHMD", + "SenseGloveBuildHacks", "SenseGloveCore", "SenseGloveLog", "SenseGloveKismet",