jackfiled/MotionCapture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: 船新的基于过程的动作绑定

Browse Source
This commit is contained in:
jackfiled 2023-03-29 22:58:07 +08:00
parent 479e64907f
commit 77bf523001
6 changed files with 226 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

295
Assets/Behaviours/BallStickBehaviour.cs
View File

@ -3,178 +3,181 @@
using UnityEngine; using UnityEngine;
using Utils.PoseTransformHandlers; using Utils.PoseTransformHandlers;
public class MainBehaviour : MonoBehaviour namespace Behaviours
{ {
private readonly GameObject[] _nodes = new GameObject[PoseLandmarkType.MaxValue]; public class BallStickBehaviour : MonoBehaviour
private readonly List<Bond> _bonds = new List<Bond>();
private readonly PoseTransform[] _poseTransforms = new PoseTransform[PoseLandmarkType.MaxValue];
private readonly UdpListener _listener = new UdpListener();
private const float Scale = 0.2f;
// Start is called before the first frame update
private void Start()
{ {
PoseTransform.AverageLength = 3; private readonly GameObject[] _nodes = new GameObject[PoseLandmarkType.MaxValue];
private readonly List<Bond> _bonds = new List<Bond>();
private readonly PoseTransform[] _poseTransforms = new PoseTransform[PoseLandmarkType.MaxValue];
private readonly UdpListener _listener = new UdpListener();
private const float Scale = 0.2f;
CreateNodes(); // Start is called before the first frame update
CreateBonds(); private void Start()
_listener.AddHandler(OnReceive);
_listener.Connect(5000);
}
// Update is called once per frame
private void Update()
{
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{ {
_nodes[i].transform.position = _poseTransforms[i].ResultPosition * -5; PoseTransform.AverageLength = 3;
CreateNodes();
CreateBonds();
_listener.AddHandler(OnReceive);
_listener.Connect(5000);
} }
foreach (var bond in _bonds) // Update is called once per frame
private void Update()
{ {
bond.UpdateBond(); for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
} {
} _nodes[i].transform.position = _poseTransforms[i].ResultPosition * -5;
}
private void OnDisable() foreach (var bond in _bonds)
{ {
_listener.DisConnect(); bond.UpdateBond();
} }
}
private void OnDisable()
{
_listener.DisConnect();
}
private void OnReceive(List<PoseLandmark> landmarks) private void OnReceive(List<PoseLandmark> landmarks)
{
foreach (var landmark in landmarks)
{ {
PoseTransform.UpdatePosition(ref _poseTransforms[landmark.Type.Value], landmark); foreach (var landmark in landmarks)
{
PoseTransform.UpdatePosition(ref _poseTransforms[landmark.Type.Value], landmark);
}
} }
}
private void CreateNodes() private void CreateNodes()
{
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{ {
var ball = GameObject.CreatePrimitive(PrimitiveType.Sphere); for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
ball.transform.localScale = new Vector3(Scale, Scale, Scale);
if (i <= 10)
{ {
ball.GetComponent<Renderer>().material.color = Color.red;//给头部添加颜色 var ball = GameObject.CreatePrimitive(PrimitiveType.Sphere);
}
else if (i <= 22)
{
ball.GetComponent<Renderer>().material.color = Color.blue;//给手部添加颜色
}
else if (i <= 32)
{
ball.GetComponent<Renderer>().material.color = Color.green;//给脚部添加颜色
}
_nodes[i] = ball; ball.transform.localScale = new Vector3(Scale, Scale, Scale);
_poseTransforms[i] = new PoseTransform(i, new AverageHandler());
if (i <= 10)
{
ball.GetComponent<Renderer>().material.color = Color.red;//给头部添加颜色
}
else if (i <= 22)
{
ball.GetComponent<Renderer>().material.color = Color.blue;//给手部添加颜色
}
else if (i <= 32)
{
ball.GetComponent<Renderer>().material.color = Color.green;//给脚部添加颜色
}
_nodes[i] = ball;
_poseTransforms[i] = new PoseTransform(i, new AverageHandler());
}
} }
}
private void CreateBonds() private void CreateBonds()
{
var headBonds = new[]
{ {
PoseLandmarkType.RightEar, var headBonds = new[]
PoseLandmarkType.RightEyeOuter, {
PoseLandmarkType.RightEye, PoseLandmarkType.RightEar,
PoseLandmarkType.RightEyeInner, PoseLandmarkType.RightEyeOuter,
PoseLandmarkType.Nose, PoseLandmarkType.RightEye,
PoseLandmarkType.LeftEyeInner, PoseLandmarkType.RightEyeInner,
PoseLandmarkType.LeftEye, PoseLandmarkType.Nose,
PoseLandmarkType.LeftEyeOuter, PoseLandmarkType.LeftEyeInner,
PoseLandmarkType.LeftEar PoseLandmarkType.LeftEye,
}; PoseLandmarkType.LeftEyeOuter,
_bonds.AddRange(GenerateBondsList(headBonds)); PoseLandmarkType.LeftEar
};
_bonds.AddRange(GenerateBondsList(headBonds));
var monthBonds = new[] var monthBonds = new[]
{ {
PoseLandmarkType.MouthLeft, PoseLandmarkType.MouthLeft,
PoseLandmarkType.MouthRight, PoseLandmarkType.MouthRight,
}; };
_bonds.AddRange(GenerateBondsList(monthBonds)); _bonds.AddRange(GenerateBondsList(monthBonds));
var leftArmBonds = new[] var leftArmBonds = new[]
{ {
PoseLandmarkType.LeftShoulder, PoseLandmarkType.LeftShoulder,
PoseLandmarkType.LeftElbow, PoseLandmarkType.LeftElbow,
PoseLandmarkType.LeftWrist, PoseLandmarkType.LeftWrist,
PoseLandmarkType.LeftPinky, PoseLandmarkType.LeftPinky,
PoseLandmarkType.LeftIndex, PoseLandmarkType.LeftIndex,
PoseLandmarkType.LeftWrist, PoseLandmarkType.LeftWrist,
PoseLandmarkType.LeftThumb PoseLandmarkType.LeftThumb
}; };
_bonds.AddRange(GenerateBondsList(leftArmBonds)); _bonds.AddRange(GenerateBondsList(leftArmBonds));
var rightArmBonds = new[] var rightArmBonds = new[]
{ {
PoseLandmarkType.RightShoulder, PoseLandmarkType.RightShoulder,
PoseLandmarkType.RightElbow, PoseLandmarkType.RightElbow,
PoseLandmarkType.RightWrist, PoseLandmarkType.RightWrist,
PoseLandmarkType.RightPinky, PoseLandmarkType.RightPinky,
PoseLandmarkType.RightIndex, PoseLandmarkType.RightIndex,
PoseLandmarkType.RightWrist, PoseLandmarkType.RightWrist,
PoseLandmarkType.RightThumb PoseLandmarkType.RightThumb
}; };
_bonds.AddRange(GenerateBondsList(rightArmBonds)); _bonds.AddRange(GenerateBondsList(rightArmBonds));
var leftLegBonds = new[] var leftLegBonds = new[]
{ {
PoseLandmarkType.LeftShoulder, PoseLandmarkType.LeftShoulder,
PoseLandmarkType.LeftHip, PoseLandmarkType.LeftHip,
PoseLandmarkType.LeftKnee, PoseLandmarkType.LeftKnee,
PoseLandmarkType.LeftAnkle, PoseLandmarkType.LeftAnkle,
PoseLandmarkType.LeftHeel, PoseLandmarkType.LeftHeel,
PoseLandmarkType.LeftFootIndex, PoseLandmarkType.LeftFootIndex,
PoseLandmarkType.LeftAnkle PoseLandmarkType.LeftAnkle
}; };
_bonds.AddRange(GenerateBondsList(leftLegBonds)); _bonds.AddRange(GenerateBondsList(leftLegBonds));
var rightLegBonds = new[] var rightLegBonds = new[]
{ {
PoseLandmarkType.RightShoulder, PoseLandmarkType.RightShoulder,
PoseLandmarkType.RightHip, PoseLandmarkType.RightHip,
PoseLandmarkType.RightKnee, PoseLandmarkType.RightKnee,
PoseLandmarkType.RightAnkle, PoseLandmarkType.RightAnkle,
PoseLandmarkType.RightHeel, PoseLandmarkType.RightHeel,
PoseLandmarkType.RightFootIndex, PoseLandmarkType.RightFootIndex,
PoseLandmarkType.RightAnkle PoseLandmarkType.RightAnkle
}; };
_bonds.AddRange(GenerateBondsList(rightLegBonds)); _bonds.AddRange(GenerateBondsList(rightLegBonds));
// 最后手动添加身体上的两条横线 // 最后手动添加身体上的两条横线
_bonds.Add(new Bond( _bonds.Add(new Bond(
_nodes[PoseLandmarkType.LeftShoulder], _nodes[PoseLandmarkType.LeftShoulder],
_nodes[PoseLandmarkType.RightShoulder], _nodes[PoseLandmarkType.RightShoulder],
Scale)); Scale));
_bonds.Add(new Bond( _bonds.Add(new Bond(
_nodes[PoseLandmarkType.LeftHip], _nodes[PoseLandmarkType.LeftHip],
_nodes[PoseLandmarkType.RightHip], _nodes[PoseLandmarkType.RightHip],
Scale));
}
/// <summary>
/// 创建棍子列表
/// </summary>
/// <param name="nodes">需要连接起来的关键点 需要按顺序设置</param>
/// <returns></returns>
private List<Bond> GenerateBondsList(int[] nodes)
{
var bonds = new List<Bond>();
for (var i = 0; i < nodes.Length - 1; i++)
{
bonds.Add(new Bond(
_nodes[nodes[i]],
_nodes[nodes[i + 1]],
Scale)); Scale));
} }
return bonds; /// <summary>
/// 创建棍子列表
/// </summary>
/// <param name="nodes">需要连接起来的关键点 需要按顺序设置</param>
/// <returns></returns>
private List<Bond> GenerateBondsList(int[] nodes)
{
var bonds = new List<Bond>();
for (var i = 0; i < nodes.Length - 1; i++)
{
bonds.Add(new Bond(
_nodes[nodes[i]],
_nodes[nodes[i + 1]],
Scale));
}
return bonds;
}
} }
} }

87
Assets/Behaviours/BoneBehaviour.cs
View File

@ -1,38 +1,35 @@
using System;
using System.Collections.Generic; using System.Collections.Generic;
using Models; using Models;
using UnityEngine; using UnityEngine;
using Utils.PoseTransformHandlers;
namespace Behaviours namespace Behaviours
{ {
public class BoneBehaviour : MonoBehaviour public class BoneBehaviour : MonoBehaviour
{ {
private readonly UdpListener _listener = new UdpListener(); private readonly UdpListener _listener = new UdpListener();
private readonly PoseTransform[] _transforms = new PoseTransform[PoseLandmarkType.MaxValue]; private Dictionary<HumanBodyBones, RotationNode> _rotationNodes;
private Animator _animator; private Animator _animator;
private void Start() private void Start()
{ {
_animator = GetComponent<Animator>(); _animator = GetComponent<Animator>();
CreateRotationNodes();
for (var i = 0; i < PoseLandmarkType.MaxValue; i++)
{
_transforms[i] = new PoseTransform(i, new ZAxisHandler());
}
_listener.AddHandler(OnReceive); _listener.AddHandler(OnReceive);
_listener.Connect(5000); _listener.Connect(5000);
} }
private void Update() private void Update()
{ {
HipUpdate(); foreach (var node in _rotationNodes)
{
RShoulderUpdate(); _animator.GetBoneTransform(node.Key).rotation = node.Value.Rotation;
}
} }
private void OnDisable() private void OnDisable()
{ {
_listener.DisConnect(); _listener.DisConnect();
@ -40,51 +37,47 @@ private void OnDisable()
private void OnReceive(List<PoseLandmark> landmarks) private void OnReceive(List<PoseLandmark> landmarks)
{ {
foreach (var landmark in landmarks) // 计算腰部
{
PoseTransform.UpdatePosition(ref _transforms[landmark.Type.Value], landmark);
}
}
private void HipUpdate()
{
var frontLeft = Vector3.Cross( var frontLeft = Vector3.Cross(
_transforms[PoseLandmarkType.RightHip] - _transforms[PoseLandmarkType.LeftHip], landmarks[PoseLandmarkType.RightHip].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3,
_transforms[PoseLandmarkType.RightShoulder] - _transforms[PoseLandmarkType.LeftHip]); landmarks[PoseLandmarkType.RightShoulder].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3);
var frontRight = Vector3.Cross( var frontRight = Vector3.Cross(
_transforms[PoseLandmarkType.LeftShoulder] - _transforms[PoseLandmarkType.RightHip], landmarks[PoseLandmarkType.LeftShoulder].Vector3 - landmarks[PoseLandmarkType.RightHip].Vector3,
_transforms[PoseLandmarkType.LeftHip] - _transforms[PoseLandmarkType.RightHip]); landmarks[PoseLandmarkType.LeftHip].Vector3 - landmarks[PoseLandmarkType.LeftHip].Vector3);
var front = frontLeft + frontRight; var front = frontLeft + frontRight;
front.Normalize(); front.Normalize();
_animator.GetBoneTransform(HumanBodyBones.Hips).rotation = Quaternion.LookRotation(front); var oldRotation = _rotationNodes[HumanBodyBones.Hips];
_rotationNodes[HumanBodyBones.Hips] = new RotationNode(
oldRotation.UnityName,
oldRotation.Rotation,
Quaternion.LookRotation(front));
} }
private void RShoulderUpdate()
{
var direction = _transforms[PoseLandmarkType.RightElbow]
- _transforms[PoseLandmarkType.RightShoulder];
var absolutelyRotation = Quaternion.LookRotation(direction);
_animator.GetBoneTransform(HumanBodyBones.RightUpperArm).rotation private void CreateRotationNodes()
= absolutelyRotation * Quaternion.Inverse(_animator.GetBoneTransform(HumanBodyBones.Hips).rotation);
}
//计算向量a到向量b的旋转角
//参数 a：起始向量; b：目标向量; n：旋转方向 (0, 1, 0)顺时针 (0, -1, 0)逆时针
private static float SignedAngleBetween(Vector3 a, Vector3 b, Vector3 n)
{ {
var angle = Vector3.Angle(a,b); _rotationNodes = new Dictionary<HumanBodyBones, RotationNode>();
var sign = Mathf.Sign(Vector3.Dot(n,Vector3.Cross(a,b)));
var signedAngle = angle * sign; var bonesArray = new[]
return (signedAngle <= 0) ? 360 + signedAngle : signedAngle; {
HumanBodyBones.Hips,
HumanBodyBones.LeftUpperArm,
HumanBodyBones.LeftLowerArm,
HumanBodyBones.RightUpperArm,
HumanBodyBones.RightLowerArm,
HumanBodyBones.LeftUpperLeg,
HumanBodyBones.LeftLowerLeg,
HumanBodyBones.RightUpperLeg,
HumanBodyBones.RightLowerLeg
};
foreach (var bone in bonesArray)
{
var rotation = _animator.GetBoneTransform(bone).rotation;
_rotationNodes.Add(bone, new RotationNode(bone, rotation, rotation));
}
} }
} }
} }

3
Assets/Models/PoseLandmark.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using UnityEngine;
namespace Models namespace Models
{ {
@ -39,6 +40,8 @@ public class PoseLandmark
/// </summary> /// </summary>
public float Z { get; } public float Z { get; }
public Vector3 Vector3 => new Vector3(X, Y, Z);
/// <summary> /// <summary>
/// 该坐标点估计的可见性 /// 该坐标点估计的可见性
/// [0,1] /// [0,1]

29
Assets/Models/RotationNode.cs Normal file
View File

@ -0,0 +1,29 @@
using UnityEngine;
namespace Models
{
public struct RotationNode
{
public HumanBodyBones UnityName;
public Quaternion PreviousRotation;
public Quaternion Rotation;
public Quaternion CalculateRotate => PreviousRotation * Quaternion.Inverse(PreviousRotation);
public RotationNode(HumanBodyBones humanBodyBone, Quaternion previousRotation, Quaternion rotation)
{
UnityName = humanBodyBone;
PreviousRotation = previousRotation;
Rotation = rotation;
}
public RotationNode(RotationNode node, PoseLandmark begin, PoseLandmark end)
{
UnityName = node.UnityName;
PreviousRotation = node.Rotation;
var forward = new Vector3(end.X - begin.X, end.Y - begin.Y, end.Z - begin.Z);
Rotation = Quaternion.LookRotation(forward);
}
}
}

3
Assets/Models/RotationNode.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 87e238cb51f34339a39945af13f43248
timeCreated: 1680090845

3
Assets/Utils/PoseTransformHandlers/ZAxisHandler.cs
View File

@ -9,7 +9,8 @@ public class ZAxisHandler : IPoseTransformHandler
public void ReceivePoseLandmark(PoseLandmark landmark) public void ReceivePoseLandmark(PoseLandmark landmark)
{ {
_result = new Vector3(landmark.X, landmark.Y, landmark.Z * 0.5f); _result = new Vector3(landmark.X, landmark.Y, landmark.Z * 0.2f);
_result = _result * -1;
} }
public Vector3 GetResultPosition() public Vector3 GetResultPosition()