From 2f2aafea65516898814bf1c9771237d41a115c8c Mon Sep 17 00:00:00 2001 From: Mamadou Babaei Date: Thu, 16 Feb 2023 18:28:34 +0100 Subject: [PATCH] virtual hand implementation with a crash from calling EvaluateSkeletalControl_AnyThread --- .../Animation/SGVirtualHandAnimInstance.cpp | 1 - .../SGVirtualHandAnimInstanceProxy.cpp | 163 +++++++++++++++++- .../SenseGlove/GameFramework/SGPawn.cpp | 1 - .../GameFramework/SGVirtualHand.cpp | 18 +- .../SGVirtualHandAnimInstanceProxy.h | 13 ++ .../SenseGlove/GameFramework/SGVirtualHand.h | 10 +- Source/SenseGlove/SenseGlove.Build.cs | 1 + 7 files changed, 195 insertions(+), 12 deletions(-) diff --git a/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstance.cpp b/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstance.cpp index a7ea49e9..0fe2e660 100644 --- a/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstance.cpp +++ b/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstance.cpp @@ -38,7 +38,6 @@ USGVirtualHandAnimInstance::USGVirtualHandAnimInstance(const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer) { - } void USGVirtualHandAnimInstance::NativeInitializeAnimation() diff --git a/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.cpp b/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.cpp index 169333da..710b239f 100644 --- a/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.cpp +++ b/Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.cpp @@ -35,6 +35,74 @@ #include "SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h" +#include "BoneContainer.h" +#include "BoneControllers/AnimNode_ModifyBone.h" +#include "BonePose.h" +#include "Containers/Array.h" +#include "Containers/Map.h" +#include "Containers/UnrealString.h" +#include "Engine/SkeletalMesh.h" +#include "Math/Rotator.h" + +#include "SenseGlove/GameFramework/SGVirtualHand.h" +#include "SGCore/SGHandPose.h" +#include "SGLog/SGLog.h" + +const FName& FSGVirtualHandAnimInstanceProxy::GetLeftHandBoneName(const int32 Finger, const int32 Joint) +{ + static const TArray> BoneNames{ + {FName(TEXT("thumb_01_l")), FName(TEXT("thumb_02_l")), FName(TEXT("thumb_03_l"))}, + {FName(TEXT("index_01_l")), FName(TEXT("index_02_l")), FName(TEXT("index_03_l"))}, + {FName(TEXT("middle_01_l")), FName(TEXT("middle_02_l")), FName(TEXT("middle_03_l"))}, + {FName(TEXT("ring_01_l")), FName(TEXT("ring_02_l")), FName(TEXT("ring_03_l"))}, + {FName(TEXT("pinky_01_l")), FName(TEXT("pinky_02_l")), FName(TEXT("pinky_03_l"))}, + }; + + // FIXME / REMOVE + // Debug code + int32 JointCount = BoneNames[Finger].Num(); + //ensureAlwaysMsgf(Joint < JointCount, TEXT("Joint index is out of bound!")); + if (Joint >= 3) + { + const FName& BoneName = BoneNames[Finger][0]; + return BoneName; + } + + ensureAlwaysMsgf(Finger < BoneNames.Num(), TEXT("Finger index is out of bound!")); + ensureAlwaysMsgf(Joint < BoneNames[Finger].Num(), TEXT("Joint index is out of bound!")); + + const FName& BoneName = BoneNames[Finger][Joint]; + return BoneName; +} + +const FName& FSGVirtualHandAnimInstanceProxy::GetRightHandBoneName(const int32 Finger, const int32 Joint) +{ + static const TArray> BoneNames{ + {FName(TEXT("thumb_01_r")), FName(TEXT("thumb_02_r")), FName(TEXT("thumb_03_r"))}, + {FName(TEXT("index_01_r")), FName(TEXT("index_02_r")), FName(TEXT("index_03_r"))}, + {FName(TEXT("middle_01_r")), FName(TEXT("middle_02_r")), FName(TEXT("middle_03_r"))}, + {FName(TEXT("ring_01_r")), FName(TEXT("ring_02_r")), FName(TEXT("ring_03_r"))}, + {FName(TEXT("pinky_01_r")), FName(TEXT("pinky_02_r")), FName(TEXT("pinky_03_r"))}, + }; + + ensureAlwaysMsgf(Finger < BoneNames.Num(), TEXT("Finger index is out of bound!")); + ensureAlwaysMsgf(Joint < BoneNames[Finger].Num(), TEXT("Joint index is out of bound!")); + + const FName& BoneName = BoneNames[Finger][Joint]; + return BoneName; +} + + +FSGVirtualHandAnimInstanceProxy::FSGVirtualHandAnimInstanceProxy() + : FAnimInstanceProxy() +{ +} + +FSGVirtualHandAnimInstanceProxy::FSGVirtualHandAnimInstanceProxy(UAnimInstance* Instance) + : FAnimInstanceProxy(Instance) +{ +} + void FSGVirtualHandAnimInstanceProxy::Initialize(UAnimInstance* InAnimInstance) { FAnimInstanceProxy::Initialize(InAnimInstance); @@ -58,11 +126,104 @@ void FSGVirtualHandAnimInstanceProxy::Update(const float DeltaSeconds) void FSGVirtualHandAnimInstanceProxy::PreEvaluateAnimation(UAnimInstance* InAnimInstance) { FAnimInstanceProxy::PreEvaluateAnimation(InAnimInstance); + + if (!IsValid(InAnimInstance)) + { + return; + } + + ASGVirtualHand* Hand(Cast(InAnimInstance->GetOwningActor())); + if (!IsValid(Hand)) + { + return; + } + + const USGHandPose* HandPose = Hand->GetHandPose(); + if (!IsValid(HandPose)) + { + return; + } + + TArray> JointRotations = HandPose->GetJointRotations(); + int32 FingerIndex = 0; + + for (const TArray& FingerJointRotations: JointRotations) + { + int32 JointIndex = 0; + + // FIXME / REMOVE + // This should not be necessary + if (JointIndex >= 3) + { + continue;; + } + + for (FRotator JointRotation: FingerJointRotations) + { + FName BoneName(Hand->IsRight() + ? GetRightHandBoneName(FingerIndex, JointIndex) + : GetLeftHandBoneName(FingerIndex, JointIndex)); + + BoneMap.Emplace(MoveTemp(BoneName), MoveTemp(JointRotation)); + + ++JointIndex; + } + + ++FingerIndex; + } } bool FSGVirtualHandAnimInstanceProxy::Evaluate(FPoseContext& Output) { - return FAnimInstanceProxy::Evaluate(Output); + const USkeletalMeshComponent* HandMeshComponent = GetSkelMeshComponent(); + if (!IsValid(HandMeshComponent)) + { + return FAnimInstanceProxy::Evaluate(Output); + } + + const USkeletalMesh* HandMesh = HandMeshComponent->SkeletalMesh; + if (!IsValid(HandMesh)) + { + return FAnimInstanceProxy::Evaluate(Output); + } + + const USkeleton* HandSkeleton = HandMesh->GetSkeleton(); + if (!IsValid(HandSkeleton)) + { + return FAnimInstanceProxy::Evaluate(Output); + } + + FComponentSpacePoseContext PoseContext(this); + PoseContext.Pose.InitPose(Output.Pose); + + for (const TTuple& Bone: BoneMap) + { + FName BoneName(Bone.Key); + FRotator BoneRotation(Bone.Value); + + FBoneReference BoneReference; + BoneReference.BoneName = BoneName; + BoneReference.BoneIndex = GetRequiredBones().GetPoseBoneIndexForBoneName(BoneName); + + FAnimNode_ModifyBone BoneController; + BoneController.BoneToModify = MoveTemp(BoneReference); + BoneController.RotationMode = EBoneModificationMode::BMM_Replace; + BoneController.RotationSpace = EBoneControlSpace::BCS_WorldSpace; + BoneController.Rotation = MoveTemp(BoneRotation); + + if (BoneController.IsValidToEvaluate(HandSkeleton, PoseContext.Pose.GetPose().GetBoneContainer())) + { + TArray BoneTransforms; + BoneController.EvaluateSkeletalControl_AnyThread(PoseContext, BoneTransforms); + + if (BoneTransforms.Num() > 0) + { + PoseContext.Pose.LocalBlendCSBoneTransforms(BoneTransforms, 1.0f); + } + } + } + + return true; } void FSGVirtualHandAnimInstanceProxy::PostUpdate(UAnimInstance* InAnimInstance) const diff --git a/Source/SenseGlove/Private/SenseGlove/GameFramework/SGPawn.cpp b/Source/SenseGlove/Private/SenseGlove/GameFramework/SGPawn.cpp index eaacd31e..b6043ed9 100644 --- a/Source/SenseGlove/Private/SenseGlove/GameFramework/SGPawn.cpp +++ b/Source/SenseGlove/Private/SenseGlove/GameFramework/SGPawn.cpp @@ -68,7 +68,6 @@ struct ASGPawn::FImpl /************************ * Methods ************************/ - }; ASGPawn::ASGPawn(const FObjectInitializer& ObjectInitializer) diff --git a/Source/SenseGlove/Private/SenseGlove/GameFramework/SGVirtualHand.cpp b/Source/SenseGlove/Private/SenseGlove/GameFramework/SGVirtualHand.cpp index 4bff2738..45dc4598 100644 --- a/Source/SenseGlove/Private/SenseGlove/GameFramework/SGVirtualHand.cpp +++ b/Source/SenseGlove/Private/SenseGlove/GameFramework/SGVirtualHand.cpp @@ -35,6 +35,7 @@ #include "SenseGlove/GameFramework/SGVirtualHand.h" +#include "Animation/Skeleton.h" #include "Components/SkeletalMeshComponent.h" #include "Engine/SkeletalMesh.h" #include "UObject/ConstructorHelpers.h" @@ -161,11 +162,6 @@ ASGVirtualHand::ASGVirtualHand(const FObjectInitializer& ObjectInitializer) Glove = nullptr; } -void ASGVirtualHand::SetHandMesh(USkeletalMesh* VirtualHandMesh, const bool bReinitPos) const -{ - Pimpl->SetHandMesh(VirtualHandMesh, bReinitPos); -} - void ASGVirtualHand::Tick(const float DeltaSeconds) { #if WITH_EDITOR @@ -211,7 +207,17 @@ void ASGVirtualHand::EditorTick(float DeltaSeconds) (void)Pimpl->CheckGlove(); } -bool ASGVirtualHand::IsGloveConnected() +USkeletalMesh* ASGVirtualHand::GetHandMesh() const +{ + return Mesh->SkeletalMesh; +} + +void ASGVirtualHand::SetHandMesh(USkeletalMesh* VirtualHandMesh, const bool bReinitPos) const +{ + Pimpl->SetHandMesh(VirtualHandMesh, bReinitPos); +} + +bool ASGVirtualHand::IsGloveConnected() const { return Pimpl->CheckGlove(); } diff --git a/Source/SenseGlove/Public/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h b/Source/SenseGlove/Public/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h index b52ab99a..2b3150e1 100644 --- a/Source/SenseGlove/Public/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h +++ b/Source/SenseGlove/Public/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h @@ -36,6 +36,8 @@ #pragma once #include "Animation/AnimInstanceProxy.h" +#include "Templates/UniquePtr.h" +#include "UObject/NameTypes.h" #include "SGVirtualHandAnimInstanceProxy.generated.h" @@ -44,6 +46,17 @@ struct SENSEGLOVE_API FSGVirtualHandAnimInstanceProxy : public FAnimInstanceProx { GENERATED_USTRUCT_BODY() +public: + static const FName& GetLeftHandBoneName(int32 Finger, int32 Joint); + static const FName& GetRightHandBoneName(int32 Finger, int32 Joint); + +public: + TMap BoneMap; + +public: + FSGVirtualHandAnimInstanceProxy(); + FSGVirtualHandAnimInstanceProxy(UAnimInstance* Instance); + protected: /** Called when our anim instance is being initialized */ virtual void Initialize(UAnimInstance* InAnimInstance) override; diff --git a/Source/SenseGlove/Public/SenseGlove/GameFramework/SGVirtualHand.h b/Source/SenseGlove/Public/SenseGlove/GameFramework/SGVirtualHand.h index d72fc516..d01e982d 100644 --- a/Source/SenseGlove/Public/SenseGlove/GameFramework/SGVirtualHand.h +++ b/Source/SenseGlove/Public/SenseGlove/GameFramework/SGVirtualHand.h @@ -41,6 +41,7 @@ #include "SGVirtualHand.generated.h" class USkeletalMesh; +class USkeleton; class USkeletalMeshComponent; class USceneComponent; @@ -128,8 +129,6 @@ public: bAnimateInEditor = bInAnimateInEditor; } - void SetHandMesh(USkeletalMesh* VirtualHandMesh, bool bReinitPos) const; - protected: FORCEINLINE void SetMesh(USkeletalMeshComponent* InMesh) { @@ -157,6 +156,11 @@ protected: virtual void EditorTick(float DeltaSeconds); public: - bool IsGloveConnected(); + USkeletalMesh* GetHandMesh() const; + void SetHandMesh(USkeletalMesh* VirtualHandMesh, bool bReinitPos) const; + + USkeleton* GetHandSkeleton() const; + + bool IsGloveConnected() const; USGHandPose* GetHandPose(); }; \ No newline at end of file diff --git a/Source/SenseGlove/SenseGlove.Build.cs b/Source/SenseGlove/SenseGlove.Build.cs index cbc0b678..d0fe82aa 100644 --- a/Source/SenseGlove/SenseGlove.Build.cs +++ b/Source/SenseGlove/SenseGlove.Build.cs @@ -52,6 +52,7 @@ public class SenseGlove : ModuleRules PrivateDependencyModuleNames.AddRange( new string[] { + "AnimGraphRuntime", "CoreUObject", "Engine", "HeadMountedDisplay",