virtual hand implementation with a crash from calling EvaluateSkeletalControl_AnyThread

Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstance.cpp

@@ -38,7 +38,6 @@
 USGVirtualHandAnimInstance::USGVirtualHandAnimInstance(const FObjectInitializer& ObjectInitializer)
     : Super(ObjectInitializer)
 {
-
 }
 
 void USGVirtualHandAnimInstance::NativeInitializeAnimation()

Source/SenseGlove/Private/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.cpp

@@ -35,6 +35,74 @@
 
 #include "SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h"
 
+#include "BoneContainer.h"
+#include "BoneControllers/AnimNode_ModifyBone.h"
+#include "BonePose.h"
+#include "Containers/Array.h"
+#include "Containers/Map.h"
+#include "Containers/UnrealString.h"
+#include "Engine/SkeletalMesh.h"
+#include "Math/Rotator.h"
+
+#include "SenseGlove/GameFramework/SGVirtualHand.h"
+#include "SGCore/SGHandPose.h"
+#include "SGLog/SGLog.h"
+
+const FName& FSGVirtualHandAnimInstanceProxy::GetLeftHandBoneName(const int32 Finger, const int32 Joint)
+{
+    static const TArray<TArray<FName>> BoneNames{
+        {FName(TEXT("thumb_01_l")), FName(TEXT("thumb_02_l")), FName(TEXT("thumb_03_l"))},
+        {FName(TEXT("index_01_l")), FName(TEXT("index_02_l")), FName(TEXT("index_03_l"))},
+        {FName(TEXT("middle_01_l")), FName(TEXT("middle_02_l")), FName(TEXT("middle_03_l"))},
+        {FName(TEXT("ring_01_l")), FName(TEXT("ring_02_l")), FName(TEXT("ring_03_l"))},
+        {FName(TEXT("pinky_01_l")), FName(TEXT("pinky_02_l")), FName(TEXT("pinky_03_l"))},
+    };
+
+    // FIXME / REMOVE
+    // Debug code
+    int32 JointCount = BoneNames[Finger].Num();
+    //ensureAlwaysMsgf(Joint < JointCount, TEXT("Joint index is out of bound!"));
+    if (Joint >= 3)
+    {
+    const FName& BoneName = BoneNames[Finger][0];
+    return BoneName;
+    }
+
+    ensureAlwaysMsgf(Finger < BoneNames.Num(), TEXT("Finger index is out of bound!"));
+    ensureAlwaysMsgf(Joint < BoneNames[Finger].Num(), TEXT("Joint index is out of bound!"));
+
+    const FName& BoneName = BoneNames[Finger][Joint];
+    return BoneName;
+}
+
+const FName& FSGVirtualHandAnimInstanceProxy::GetRightHandBoneName(const int32 Finger, const int32 Joint)
+{
+    static const TArray<TArray<FName>> BoneNames{
+        {FName(TEXT("thumb_01_r")), FName(TEXT("thumb_02_r")), FName(TEXT("thumb_03_r"))},
+        {FName(TEXT("index_01_r")), FName(TEXT("index_02_r")), FName(TEXT("index_03_r"))},
+        {FName(TEXT("middle_01_r")), FName(TEXT("middle_02_r")), FName(TEXT("middle_03_r"))},
+        {FName(TEXT("ring_01_r")), FName(TEXT("ring_02_r")), FName(TEXT("ring_03_r"))},
+        {FName(TEXT("pinky_01_r")), FName(TEXT("pinky_02_r")), FName(TEXT("pinky_03_r"))},
+    };
+
+    ensureAlwaysMsgf(Finger < BoneNames.Num(), TEXT("Finger index is out of bound!"));
+    ensureAlwaysMsgf(Joint < BoneNames[Finger].Num(), TEXT("Joint index is out of bound!"));
+
+    const FName& BoneName = BoneNames[Finger][Joint];
+    return BoneName;
+}
+
+
+FSGVirtualHandAnimInstanceProxy::FSGVirtualHandAnimInstanceProxy()
+    : FAnimInstanceProxy()
+{
+}
+
+FSGVirtualHandAnimInstanceProxy::FSGVirtualHandAnimInstanceProxy(UAnimInstance* Instance)
+    : FAnimInstanceProxy(Instance)
+{
+}
+
 void FSGVirtualHandAnimInstanceProxy::Initialize(UAnimInstance* InAnimInstance)
 {
     FAnimInstanceProxy::Initialize(InAnimInstance);
@@ -58,11 +126,104 @@ void FSGVirtualHandAnimInstanceProxy::Update(const float DeltaSeconds)
 void FSGVirtualHandAnimInstanceProxy::PreEvaluateAnimation(UAnimInstance* InAnimInstance)
 {
     FAnimInstanceProxy::PreEvaluateAnimation(InAnimInstance);
+
+    if (!IsValid(InAnimInstance))
+    {
+        return;
+    }
+
+    ASGVirtualHand* Hand(Cast<ASGVirtualHand>(InAnimInstance->GetOwningActor()));
+    if (!IsValid(Hand))
+    {
+        return;
+    }
+
+    const USGHandPose* HandPose = Hand->GetHandPose();
+    if (!IsValid(HandPose))
+    {
+        return;
+    }
+
+    TArray<TArray<FRotator>> JointRotations = HandPose->GetJointRotations();
+    int32 FingerIndex = 0;
+
+    for (const TArray<FRotator>& FingerJointRotations: JointRotations)
+    {
+        int32 JointIndex = 0;
+
+        // FIXME / REMOVE
+        // This should not be necessary
+        if (JointIndex >= 3)
+        {
+            continue;;
+        }
+
+        for (FRotator JointRotation: FingerJointRotations)
+        {
+            FName BoneName(Hand->IsRight()
+                ? GetRightHandBoneName(FingerIndex, JointIndex)
+                : GetLeftHandBoneName(FingerIndex, JointIndex));
+
+            BoneMap.Emplace(MoveTemp(BoneName), MoveTemp(JointRotation));
+
+            ++JointIndex;
+        }
+
+        ++FingerIndex;
+    }
 }
 
 bool FSGVirtualHandAnimInstanceProxy::Evaluate(FPoseContext& Output)
 {
-    return FAnimInstanceProxy::Evaluate(Output);
+    const USkeletalMeshComponent* HandMeshComponent = GetSkelMeshComponent();
+    if (!IsValid(HandMeshComponent))
+    {
+        return FAnimInstanceProxy::Evaluate(Output);
+    }
+
+    const USkeletalMesh* HandMesh = HandMeshComponent->SkeletalMesh;
+    if (!IsValid(HandMesh))
+    {
+        return FAnimInstanceProxy::Evaluate(Output);
+    }
+
+    const USkeleton* HandSkeleton = HandMesh->GetSkeleton();
+    if (!IsValid(HandSkeleton))
+    {
+        return FAnimInstanceProxy::Evaluate(Output);
+    }
+
+    FComponentSpacePoseContext PoseContext(this);
+    PoseContext.Pose.InitPose(Output.Pose);
+
+    for (const TTuple<FName, FRotator>& Bone: BoneMap)
+    {
+        FName BoneName(Bone.Key);
+        FRotator BoneRotation(Bone.Value);
+
+        FBoneReference BoneReference;
+        BoneReference.BoneName = BoneName;
+        BoneReference.BoneIndex = GetRequiredBones().GetPoseBoneIndexForBoneName(BoneName);
+
+        FAnimNode_ModifyBone BoneController;
+        BoneController.BoneToModify = MoveTemp(BoneReference);
+        BoneController.RotationMode = EBoneModificationMode::BMM_Replace;
+        BoneController.RotationSpace = EBoneControlSpace::BCS_WorldSpace;
+        BoneController.Rotation = MoveTemp(BoneRotation);
+
+        if (BoneController.IsValidToEvaluate(HandSkeleton, PoseContext.Pose.GetPose().GetBoneContainer()))
+        {
+            TArray<FBoneTransform> BoneTransforms;
+            BoneController.EvaluateSkeletalControl_AnyThread(PoseContext, BoneTransforms);
+
+            if (BoneTransforms.Num() > 0)
+            {
+                PoseContext.Pose.LocalBlendCSBoneTransforms(BoneTransforms, 1.0f);
+            }
+        }
+    }
+
+    return true;
 }
 
 void FSGVirtualHandAnimInstanceProxy::PostUpdate(UAnimInstance* InAnimInstance) const

Source/SenseGlove/Private/SenseGlove/GameFramework/SGPawn.cpp

@@ -68,7 +68,6 @@ struct ASGPawn::FImpl
     /************************
      * Methods
      ************************/
-
 };
 
 ASGPawn::ASGPawn(const FObjectInitializer& ObjectInitializer)

Source/SenseGlove/Private/SenseGlove/GameFramework/SGVirtualHand.cpp

@@ -35,6 +35,7 @@
 
 #include "SenseGlove/GameFramework/SGVirtualHand.h"
 
+#include "Animation/Skeleton.h"
 #include "Components/SkeletalMeshComponent.h"
 #include "Engine/SkeletalMesh.h"
 #include "UObject/ConstructorHelpers.h"
@@ -161,11 +162,6 @@ ASGVirtualHand::ASGVirtualHand(const FObjectInitializer& ObjectInitializer)
     Glove = nullptr;
 }
 
-void ASGVirtualHand::SetHandMesh(USkeletalMesh* VirtualHandMesh, const bool bReinitPos) const
-{
-    Pimpl->SetHandMesh(VirtualHandMesh, bReinitPos);
-}
-
 void ASGVirtualHand::Tick(const float DeltaSeconds)
 {
 #if WITH_EDITOR
@@ -211,7 +207,17 @@ void ASGVirtualHand::EditorTick(float DeltaSeconds)
     (void)Pimpl->CheckGlove();
 }
 
-bool ASGVirtualHand::IsGloveConnected()
+USkeletalMesh* ASGVirtualHand::GetHandMesh() const
+{
+    return Mesh->SkeletalMesh;
+}
+
+void ASGVirtualHand::SetHandMesh(USkeletalMesh* VirtualHandMesh, const bool bReinitPos) const
+{
+    Pimpl->SetHandMesh(VirtualHandMesh, bReinitPos);
+}
+
+bool ASGVirtualHand::IsGloveConnected() const
 {
     return Pimpl->CheckGlove();
 }

Source/SenseGlove/Public/SenseGlove/Animation/SGVirtualHandAnimInstanceProxy.h

@@ -36,6 +36,8 @@
 #pragma once
 
 #include "Animation/AnimInstanceProxy.h"
+#include "Templates/UniquePtr.h"
+#include "UObject/NameTypes.h"
 
 #include "SGVirtualHandAnimInstanceProxy.generated.h"
 
@@ -44,6 +46,17 @@ struct SENSEGLOVE_API FSGVirtualHandAnimInstanceProxy : public FAnimInstanceProx
 {
 	GENERATED_USTRUCT_BODY()
 
+public:
+    static const FName& GetLeftHandBoneName(int32 Finger, int32 Joint);
+    static const FName& GetRightHandBoneName(int32 Finger, int32 Joint);
+
+public:
+    TMap<FName, FRotator> BoneMap;
+
+public:
+	FSGVirtualHandAnimInstanceProxy();
+	FSGVirtualHandAnimInstanceProxy(UAnimInstance* Instance);
+
 protected:
 	/** Called when our anim instance is being initialized */
 	virtual void Initialize(UAnimInstance* InAnimInstance) override;

Source/SenseGlove/Public/SenseGlove/GameFramework/SGVirtualHand.h

@@ -41,6 +41,7 @@
 #include "SGVirtualHand.generated.h"
 
 class USkeletalMesh;
+class USkeleton;
 class USkeletalMeshComponent;
 class USceneComponent;
 
@@ -128,8 +129,6 @@ public:
         bAnimateInEditor = bInAnimateInEditor;
     }
 
-    void SetHandMesh(USkeletalMesh* VirtualHandMesh, bool bReinitPos) const;
-
 protected:
     FORCEINLINE void SetMesh(USkeletalMeshComponent* InMesh)
     {
@@ -157,6 +156,11 @@ protected:
     virtual void EditorTick(float DeltaSeconds);
 
 public:
-    bool IsGloveConnected();
+    USkeletalMesh* GetHandMesh() const;
+    void SetHandMesh(USkeletalMesh* VirtualHandMesh, bool bReinitPos) const;
+
+    USkeleton* GetHandSkeleton() const;
+
+    bool IsGloveConnected() const;
     USGHandPose* GetHandPose();
 };

Source/SenseGlove/SenseGlove.Build.cs

@@ -52,6 +52,7 @@ public class SenseGlove : ModuleRules
         PrivateDependencyModuleNames.AddRange(
             new string[]
             {
+                "AnimGraphRuntime",
                 "CoreUObject",
                 "Engine",
                 "HeadMountedDisplay",