post-guard/MotionCapture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: 船新的基于过程的动作绑定

Browse Source
This commit is contained in:
jackfiled 2023-03-29 22:58:07 +08:00
parent 479e64907f
commit 77bf523001
6 changed files with 226 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

295
Assets/Behaviours/BallStickBehaviour.cs
View File

@ -3,178 +3,181 @@
using UnityEngine; using UnityEngine;
using Utils.PoseTransformHandlers; using Utils.PoseTransformHandlers;
public class MainBehaviour : MonoBehaviour namespace Behaviours
{ {
private readonly GameObject[] _nodes = new GameObject[PoseLandmarkType.MaxValue]; public class BallStickBehaviour : MonoBehaviour
private readonly List<Bond> _bonds = new List<Bond>();
private readonly PoseTransform[] _poseTransforms = new PoseTransform[PoseLandmarkType.MaxValue];
private readonly UdpListener _listener = new UdpListener();
private const float Scale = 0.2f;
// Start is called before the first frame update
private void Start()
{ {
PoseTransform.AverageLength = 3; private readonly GameObject[] _nodes = new GameObject[PoseLandmarkType.MaxValue];
private readonly List<Bond> _bonds = new List<Bond>();
private readonly PoseTransform[] _poseTransforms = new PoseTransform[PoseLandmarkType.MaxValue];
private readonly UdpListener _listener = new UdpListener();
private const float Scale = 0.2f;
CreateNodes(); // Start is called before the first frame update
CreateBonds(); private void Start()
_listener.AddHandler(OnReceive);
_listener.Connect(5000);
}
// Update is called once per frame
private void Update()
{
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{ {
_nodes[i].transform.position = _poseTransforms[i].ResultPosition * -5; PoseTransform.AverageLength = 3;
CreateNodes();
CreateBonds();
_listener.AddHandler(OnReceive);
_listener.Connect(5000);
} }
foreach (var bond in _bonds) // Update is called once per frame
private void Update()
{ {
bond.UpdateBond(); for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
}
}
private void OnDisable()
{
_listener.DisConnect();
}
private void OnReceive(List<PoseLandmark> landmarks)
{
foreach (var landmark in landmarks)
{
PoseTransform.UpdatePosition(ref _poseTransforms[landmark.Type.Value], landmark);
}
}
private void CreateNodes()
{
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{
var ball = GameObject.CreatePrimitive(PrimitiveType.Sphere);
ball.transform.localScale = new Vector3(Scale, Scale, Scale);
if (i <= 10)
{ {
ball.GetComponent<Renderer>().material.color = Color.red;//给头部添加颜色 _nodes[i].transform.position = _poseTransforms[i].ResultPosition * -5;
}
else if (i <= 22)
{
ball.GetComponent<Renderer>().material.color = Color.blue;//给手部添加颜色
}
else if (i <= 32)
{
ball.GetComponent<Renderer>().material.color = Color.green;//给脚部添加颜色
} }
_nodes[i] = ball; foreach (var bond in _bonds)
_poseTransforms[i] = new PoseTransform(i, new AverageHandler()); {
bond.UpdateBond();
}
} }
}
private void CreateBonds() private void OnDisable()
{
var headBonds = new[]
{ {
PoseLandmarkType.RightEar, _listener.DisConnect();
PoseLandmarkType.RightEyeOuter, }
PoseLandmarkType.RightEye,
PoseLandmarkType.RightEyeInner,
PoseLandmarkType.Nose,
PoseLandmarkType.LeftEyeInner,
PoseLandmarkType.LeftEye,
PoseLandmarkType.LeftEyeOuter,
PoseLandmarkType.LeftEar
};
_bonds.AddRange(GenerateBondsList(headBonds));
var monthBonds = new[] private void OnReceive(List<PoseLandmark> landmarks)
{ {
PoseLandmarkType.MouthLeft, foreach (var landmark in landmarks)
PoseLandmarkType.MouthRight, {
}; PoseTransform.UpdatePosition(ref _poseTransforms[landmark.Type.Value], landmark);
_bonds.AddRange(GenerateBondsList(monthBonds)); }
}
var leftArmBonds = new[] private void CreateNodes()
{ {
PoseLandmarkType.LeftShoulder, for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
PoseLandmarkType.LeftElbow, {
PoseLandmarkType.LeftWrist, var ball = GameObject.CreatePrimitive(PrimitiveType.Sphere);
PoseLandmarkType.LeftPinky,
PoseLandmarkType.LeftIndex,
PoseLandmarkType.LeftWrist,
PoseLandmarkType.LeftThumb
};
_bonds.AddRange(GenerateBondsList(leftArmBonds));
var rightArmBonds = new[] ball.transform.localScale = new Vector3(Scale, Scale, Scale);
if (i <= 10)
{
ball.GetComponent<Renderer>().material.color = Color.red;//给头部添加颜色
}
else if (i <= 22)
{
ball.GetComponent<Renderer>().material.color = Color.blue;//给手部添加颜色
}
else if (i <= 32)
{
ball.GetComponent<Renderer>().material.color = Color.green;//给脚部添加颜色
}
_nodes[i] = ball;
_poseTransforms[i] = new PoseTransform(i, new AverageHandler());
}
}
private void CreateBonds()
{ {
PoseLandmarkType.RightShoulder, var headBonds = new[]
PoseLandmarkType.RightElbow, {
PoseLandmarkType.RightWrist, PoseLandmarkType.RightEar,
PoseLandmarkType.RightPinky, PoseLandmarkType.RightEyeOuter,
PoseLandmarkType.RightIndex, PoseLandmarkType.RightEye,
PoseLandmarkType.RightWrist, PoseLandmarkType.RightEyeInner,
PoseLandmarkType.RightThumb PoseLandmarkType.Nose,
}; PoseLandmarkType.LeftEyeInner,
_bonds.AddRange(GenerateBondsList(rightArmBonds)); PoseLandmarkType.LeftEye,
PoseLandmarkType.LeftEyeOuter,
PoseLandmarkType.LeftEar
};
_bonds.AddRange(GenerateBondsList(headBonds));
var leftLegBonds = new[] var monthBonds = new[]
{ {
PoseLandmarkType.LeftShoulder, PoseLandmarkType.MouthLeft,
PoseLandmarkType.LeftHip, PoseLandmarkType.MouthRight,
PoseLandmarkType.LeftKnee, };
PoseLandmarkType.LeftAnkle, _bonds.AddRange(GenerateBondsList(monthBonds));
PoseLandmarkType.LeftHeel,
PoseLandmarkType.LeftFootIndex,
PoseLandmarkType.LeftAnkle
};
_bonds.AddRange(GenerateBondsList(leftLegBonds));
var rightLegBonds = new[] var leftArmBonds = new[]
{ {
PoseLandmarkType.RightShoulder, PoseLandmarkType.LeftShoulder,
PoseLandmarkType.RightHip, PoseLandmarkType.LeftElbow,
PoseLandmarkType.RightKnee, PoseLandmarkType.LeftWrist,
PoseLandmarkType.RightAnkle, PoseLandmarkType.LeftPinky,
PoseLandmarkType.RightHeel, PoseLandmarkType.LeftIndex,
PoseLandmarkType.RightFootIndex, PoseLandmarkType.LeftWrist,
PoseLandmarkType.RightAnkle PoseLandmarkType.LeftThumb
}; };
_bonds.AddRange(GenerateBondsList(rightLegBonds)); _bonds.AddRange(GenerateBondsList(leftArmBonds));
// 最后手动添加身体上的两条横线 var rightArmBonds = new[]
_bonds.Add(new Bond( {
_nodes[PoseLandmarkType.LeftShoulder], PoseLandmarkType.RightShoulder,
_nodes[PoseLandmarkType.RightShoulder], PoseLandmarkType.RightElbow,
Scale)); PoseLandmarkType.RightWrist,
_bonds.Add(new Bond( PoseLandmarkType.RightPinky,
_nodes[PoseLandmarkType.LeftHip], PoseLandmarkType.RightIndex,
_nodes[PoseLandmarkType.RightHip], PoseLandmarkType.RightWrist,
Scale)); PoseLandmarkType.RightThumb
} };
_bonds.AddRange(GenerateBondsList(rightArmBonds));
/// <summary> var leftLegBonds = new[]
/// 创建棍子列表 {
/// </summary> PoseLandmarkType.LeftShoulder,
/// <param name="nodes">需要连接起来的关键点 需要按顺序设置</param> PoseLandmarkType.LeftHip,
/// <returns></returns> PoseLandmarkType.LeftKnee,
private List<Bond> GenerateBondsList(int[] nodes) PoseLandmarkType.LeftAnkle,
{ PoseLandmarkType.LeftHeel,
var bonds = new List<Bond>(); PoseLandmarkType.LeftFootIndex,
PoseLandmarkType.LeftAnkle
};
_bonds.AddRange(GenerateBondsList(leftLegBonds));
for (var i = 0; i < nodes.Length - 1; i++) var rightLegBonds = new[]
{ {
bonds.Add(new Bond( PoseLandmarkType.RightShoulder,
_nodes[nodes[i]], PoseLandmarkType.RightHip,
_nodes[nodes[i + 1]], PoseLandmarkType.RightKnee,
PoseLandmarkType.RightAnkle,
PoseLandmarkType.RightHeel,
PoseLandmarkType.RightFootIndex,
PoseLandmarkType.RightAnkle
};
_bonds.AddRange(GenerateBondsList(rightLegBonds));
// 最后手动添加身体上的两条横线
_bonds.Add(new Bond(
_nodes[PoseLandmarkType.LeftShoulder],
_nodes[PoseLandmarkType.RightShoulder],
Scale));
_bonds.Add(new Bond(
_nodes[PoseLandmarkType.LeftHip],
_nodes[PoseLandmarkType.RightHip],
Scale)); Scale));
} }
return bonds; /// <summary>
/// 创建棍子列表
/// </summary>
/// <param name="nodes">需要连接起来的关键点 需要按顺序设置</param>
/// <returns></returns>
private List<Bond> GenerateBondsList(int[] nodes)
{
var bonds = new List<Bond>();
for (var i = 0; i < nodes.Length - 1; i++)
{
bonds.Add(new Bond(
_nodes[nodes[i]],
_nodes[nodes[i + 1]],
Scale));
}
return bonds;
}
} }
} }

81
Assets/Behaviours/BoneBehaviour.cs
View File

@ -1,26 +1,20 @@
using System;
using System.Collections.Generic; using System.Collections.Generic;
using Models; using Models;
using UnityEngine; using UnityEngine;
using Utils.PoseTransformHandlers;
namespace Behaviours namespace Behaviours
{ {
public class BoneBehaviour : MonoBehaviour public class BoneBehaviour : MonoBehaviour
{ {
private readonly UdpListener _listener = new UdpListener(); private readonly UdpListener _listener = new UdpListener();
private readonly PoseTransform[] _transforms = new PoseTransform[PoseLandmarkType.MaxValue]; private Dictionary<HumanBodyBones, RotationNode> _rotationNodes;
private Animator _animator; private Animator _animator;
private void Start() private void Start()
{ {
_animator = GetComponent<Animator>(); _animator = GetComponent<Animator>();
CreateRotationNodes();
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{
_transforms[i] = new PoseTransform(i, new ZAxisHandler());
}
_listener.AddHandler(OnReceive); _listener.AddHandler(OnReceive);
_listener.Connect(5000); _listener.Connect(5000);
@ -28,11 +22,14 @@ private void Start()
private void Update() private void Update()
{ {
HipUpdate(); foreach (var node in _rotationNodes)
{
RShoulderUpdate(); _animator.GetBoneTransform(node.Key).rotation = node.Value.Rotation;
}
} }
private void OnDisable() private void OnDisable()
{ {
_listener.DisConnect(); _listener.DisConnect();
@ -40,51 +37,47 @@ private void OnDisable()
private void OnReceive(List<PoseLandmark> landmarks) private void OnReceive(List<PoseLandmark> landmarks)
{ {
foreach (var landmark in landmarks) // 计算腰部
{
PoseTransform.UpdatePosition(ref _transforms[landmark.Type.Value], landmark);
}
}
private void HipUpdate()
{
var frontLeft = Vector3.Cross( var frontLeft = Vector3.Cross(
_transforms[PoseLandmarkType.RightHip] - _transforms[PoseLandmarkType.LeftHip], landmarks[PoseLandmarkType.RightHip].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3,
_transforms[PoseLandmarkType.RightShoulder] - _transforms[PoseLandmarkType.LeftHip]); landmarks[PoseLandmarkType.RightShoulder].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3);
var frontRight = Vector3.Cross( var frontRight = Vector3.Cross(
_transforms[PoseLandmarkType.LeftShoulder] - _transforms[PoseLandmarkType.RightHip], landmarks[PoseLandmarkType.LeftShoulder].Vector3 - landmarks[PoseLandmarkType.RightHip].Vector3,
_transforms[PoseLandmarkType.LeftHip] - _transforms[PoseLandmarkType.RightHip]); landmarks[PoseLandmarkType.LeftHip].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3);
var front = frontLeft + frontRight; var front = frontLeft + frontRight;
front.Normalize(); front.Normalize();
_animator.GetBoneTransform(HumanBodyBones.Hips).rotation = Quaternion.LookRotation(front); var oldRotation = _rotationNodes[HumanBodyBones.Hips];
_rotationNodes[HumanBodyBones.Hips] = new RotationNode(
oldRotation.UnityName,
oldRotation.Rotation,
Quaternion.LookRotation(front));
} }
private void RShoulderUpdate() private void CreateRotationNodes()
{ {
var direction = _transforms[PoseLandmarkType.RightElbow] _rotationNodes = new Dictionary<HumanBodyBones, RotationNode>();
- _transforms[PoseLandmarkType.RightShoulder];
var absolutelyRotation = Quaternion.LookRotation(direction);
_animator.GetBoneTransform(HumanBodyBones.RightUpperArm).rotation var bonesArray = new[]
= absolutelyRotation * Quaternion.Inverse(_animator.GetBoneTransform(HumanBodyBones.Hips).rotation); {
} HumanBodyBones.Hips,
HumanBodyBones.LeftUpperArm,
HumanBodyBones.LeftLowerArm,
HumanBodyBones.RightUpperArm,
HumanBodyBones.RightLowerArm,
HumanBodyBones.LeftUpperLeg,
HumanBodyBones.LeftLowerLeg,
HumanBodyBones.RightUpperLeg,
HumanBodyBones.RightLowerLeg
};
foreach (var bone in bonesArray)
//计算向量a到向量b的旋转角 {
//参数 a：起始向量; b：目标向量; n：旋转方向 (0, 1, 0)顺时针 (0, -1, 0)逆时针 var rotation = _animator.GetBoneTransform(bone).rotation;
private static float SignedAngleBetween(Vector3 a, Vector3 b, Vector3 n) _rotationNodes.Add(bone, new RotationNode(bone, rotation, rotation));
{ }
var angle = Vector3.Angle(a,b);
var sign = Mathf.Sign(Vector3.Dot(n,Vector3.Cross(a,b)));
var signedAngle = angle * sign;
return (signedAngle <= 0) ? 360 + signedAngle : signedAngle;
} }
} }
} }

3
Assets/Models/PoseLandmark.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using UnityEngine;
namespace Models namespace Models
{ {
@ -39,6 +40,8 @@ public class PoseLandmark
/// </summary> /// </summary>
public float Z { get; } public float Z { get; }
public Vector3 Vector3 => new Vector3(X, Y, Z);
/// <summary> /// <summary>
/// 该坐标点估计的可见性 /// 该坐标点估计的可见性
/// [0,1] /// [0,1]

29
Assets/Models/RotationNode.cs Normal file
View File

@ -0,0 +1,29 @@
using UnityEngine;
namespace Models
{
public struct RotationNode
{
public HumanBodyBones UnityName;
public Quaternion PreviousRotation;
public Quaternion Rotation;
public Quaternion CalculateRotate => PreviousRotation * Quaternion.Inverse(PreviousRotation);
public RotationNode(HumanBodyBones humanBodyBone, Quaternion previousRotation, Quaternion rotation)
{
UnityName = humanBodyBone;
PreviousRotation = previousRotation;
Rotation = rotation;
}
public RotationNode(RotationNode node, PoseLandmark begin, PoseLandmark end)
{
UnityName = node.UnityName;
PreviousRotation = node.Rotation;
var forward = new Vector3(end.X - begin.X, end.Y - begin.Y, end.Z - begin.Z);
Rotation = Quaternion.LookRotation(forward);
}
}
}

3
Assets/Models/RotationNode.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 87e238cb51f34339a39945af13f43248
timeCreated: 1680090845

3
Assets/Utils/PoseTransformHandlers/ZAxisHandler.cs
View File

@ -9,7 +9,8 @@ public class ZAxisHandler : IPoseTransformHandler
public void ReceivePoseLandmark(PoseLandmark landmark) public void ReceivePoseLandmark(PoseLandmark landmark)
{ {
_result = new Vector3(landmark.X, landmark.Y, landmark.Z * 0.5f); _result = new Vector3(landmark.X, landmark.Y, landmark.Z * 0.2f);
_result = _result * -1;
} }
public Vector3 GetResultPosition() public Vector3 GetResultPosition()