Compare commits

..

No commits in common. "5e09379b4ea8cfd00e92fa5347f49aeff3577d45" and "371fc779f2c3e7da0b8261fef6b42dd05334d2f9" have entirely different histories.

24 changed files with 25 additions and 392 deletions

View File

@ -1,17 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="deploymentTargetDropDown">
<runningDeviceTargetSelectedWithDropDown>
<targetSelectedWithDropDown>
<Target>
<type value="RUNNING_DEVICE_TARGET" />
<type value="QUICK_BOOT_TARGET" />
<deviceKey>
<Key>
<type value="SERIAL_NUMBER" />
<value value="adb-3040223527000PT-yH8BGJ._adb-tls-connect._tcp" />
<type value="VIRTUAL_DEVICE_PATH" />
<value value="$USER_HOME$/.android/avd/Pixel_3a_API_33_x86_64.avd" />
</Key>
</deviceKey>
</Target>
</runningDeviceTargetSelectedWithDropDown>
<timeTargetWasSelectedWithDropDown value="2023-01-17T14:13:43.027203Z" />
</targetSelectedWithDropDown>
<timeTargetWasSelectedWithDropDown value="2023-01-10T13:25:57.409214Z" />
</component>
</project>

View File

@ -7,7 +7,6 @@
<option name="testRunner" value="GRADLE" />
<option name="distributionType" value="DEFAULT_WRAPPED" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="gradleJvm" value="Embedded JDK" />
<option name="modules">
<set>
<option value="$PROJECT_DIR$" />

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="Android Studio default JDK" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/build/classes" />
</component>
<component name="ProjectType">

View File

@ -1,21 +0,0 @@
# CHANGELOG
### 2023-1-7
- 创建git仓库
- 参考[GitHub - LeeJeongHwi/Mediapipe_pose_Tracking_AAR_example: Mediapipe Pose Tracking AAR (android) Example](https://github.com/LeeJeongHwi/Mediapipe_pose_Tracking_AAR_example),使用`Kotlin`开始开发。
- 基本实现动作捕捉的问题。
### 2023-1-11
- 参考[how to switch cameras on Android? · Issue #1842 · google/mediapipe · GitHub](https://github.com/google/mediapipe/issues/1842)添加翻转摄像头功能。
### 2023-1-17
- 自行编译`MediaPipe`框架,解决获取捕捉数据的问题。
### 2023-1-19
- 参考[Pose landmarks: segmentation seems to be enabled by default · Issue #2454 · google/mediapipe · GitHub](https://github.com/google/mediapipe/issues/2454),解决自带人体遮罩的问题。

View File

@ -1,23 +0,0 @@
# MotionCaptureAndroid
动作捕捉系统数据采集Android应用程序
基于Google开源机器学习方案[MediaPipe](https://google.github.io/mediapipe/)中的`Pose`解决方案开发。
## 开发
开发使用的IDE
- [Android Studio](https://developer.android.google.cn/studio/)
使用`Android Studio`导入项目即可。
其中编译`MediaPipe`相关`AAR`组件的步骤详见[编译MediaPipe框架 - Ricardo的博客](https://rrricardo.top/blog/2022/11/11/compile-mediapipe/)。
### TODO
- ~~MediaPipe相关功能实现~~
- 局域网里主机的发现
- 同主机之间的高效数据传输

View File

@ -34,15 +34,13 @@ android {
dependencies {
// Android
implementation 'androidx.core:core-ktx:1.9.0'
implementation 'androidx.appcompat:appcompat:1.5.1'
implementation 'com.google.android.material:material:1.7.0'
implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
implementation 'androidx.core:core-ktx:latest.release'
implementation 'androidx.core:core-ktx:+'
implementation 'androidx.core:core-ktx:1.7.0'
implementation 'androidx.appcompat:appcompat:1.4.1'
implementation 'com.google.android.material:material:1.5.0'
implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
testImplementation 'junit:junit:4.13.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
// MediaPipe依赖文件
implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
@ -51,7 +49,7 @@ dependencies {
implementation 'com.google.flogger:flogger-system-backend:latest.release'
implementation 'com.google.code.findbugs:jsr305:latest.release'
implementation 'com.google.guava:guava:27.0.1-android'
implementation 'com.google.protobuf:protobuf-javalite:latest.release'
implementation 'com.google.protobuf:protobuf-javalite:3.19.1'
// CameraX core library
def camerax_version = "1.2.0"
implementation "androidx.camera:camera-core:$camerax_version"

Binary file not shown.

View File

@ -3,8 +3,6 @@
xmlns:tools="http://schemas.android.com/tools">
<!-- For using the Camera -->
<uses-permission android:name="android.permission.CAMERA" />
<!-- For sending data -->
<uses-permission android:name="android.permission.INTERNET"/>
<uses-feature android:name="android.hardware.camera" />
<!-- For mediapipe -->

Binary file not shown.

Binary file not shown.

BIN
app/src/main/assets/pose_tracking_gpu.binarypb Executable file → Normal file

Binary file not shown.

View File

@ -6,10 +6,7 @@ import android.os.Bundle
import android.util.Log
import android.util.Size
import android.view.*
import android.widget.EditText
import android.widget.Toast
import androidx.appcompat.widget.Toolbar
import androidx.fragment.app.DialogFragment
import com.google.mediapipe.components.CameraHelper.CameraFacing
import com.google.mediapipe.components.CameraXPreviewHelper
import com.google.mediapipe.components.ExternalTextureConverter
@ -18,13 +15,10 @@ import com.google.mediapipe.components.PermissionHelper
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList
import com.google.mediapipe.framework.AndroidAssetUtil
import com.google.mediapipe.framework.PacketGetter
import com.google.mediapipe.framework.ProtoUtil
import com.google.mediapipe.glutil.EglManager
import com.google.protobuf.InvalidProtocolBufferException
import top.rrricardo.motioncapture.models.PoseLandmark
import java.nio.ByteBuffer
class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListener {
class MainActivity : AppCompatActivity() {
// 一些在设置MediaPipe时会用到的字符串常量
private val tag = "MainActivity"
private val binaryGraphName = "pose_tracking_gpu.binarypb"
@ -39,14 +33,13 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
lateinit var converter: ExternalTextureConverter
lateinit var cameraHelper: CameraXPreviewHelper
private var cameraFacing = CameraFacing.BACK
private var isSet = false
private lateinit var sender: UdpSender
private lateinit var toolbar: Toolbar
init {
// 加载项目中用到的jni库
System.loadLibrary("mediapipe_jni")
System.loadLibrary("opencv_java3")
}
override fun onCreate(savedInstanceState: Bundle?) {
@ -62,13 +55,7 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
Log.i(tag, "Flip Camera Button Pressed.")
flipCamera()
true
}
R.id.action_show_SetIpPortDialog -> {
Log.i(tag, "show SetIpPortDialog Button Pressed.")
showPortSetDialog()
true
}
else -> {
} else -> {
super.onMenuItemSelected(it.itemId, it)
}
}
@ -92,37 +79,15 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
processor.videoSurfaceOutput.setFlipY(true)
// 捕获获得的坐标数据包
// 遇到InvalidProtocolBufferException
// 参考 https://github.com/google/mediapipe/issues/1013
ProtoUtil.registerTypeName(NormalizedLandmarkList::class.java,
"mediapipe.NormalizedLandmarkList")
// 展示
processor.addPacketCallback(
outputLandmarksStreamName
) {
Log.i(tag, "Received Landmark Packets.")
try {
Log.i(tag, "Receive Pose Packet.")
// 无法采用这种方法获取packet
// 回报Fatal signal 6 (SIGABRT), code -1 (SI_QUEUE) in tid 15943 (Thread-9), pid 15810 (o.motioncapture)
/*val packetRaw = PacketGetter.getBytes(it)
val landmarks = NormalizedLandmarkList.parseFrom(packetRaw)
Log.i(tag, getPoseLandmarksDebugString(landmarks))*/
val landmarks = PacketGetter.getProto(it,
NormalizedLandmarkList.getDefaultInstance())
val poseLandmarks = PoseLandmark.valueOf(landmarks, System.currentTimeMillis())
val buffer = ByteBuffer.allocate(
poseLandmarks.size * PoseLandmark.packetLength)
for (poseLandmark in poseLandmarks) {
buffer.put(poseLandmark.toByteArray())
}
if (isSet) {
sender.sendMessage(buffer.array())
}
val landmarkRaw = PacketGetter.getProtoBytes(it)
val poseLandmarks = NormalizedLandmarkList.parseFrom(landmarkRaw)
// 从这里可以获得每个特征点的座标
} catch (exception: InvalidProtocolBufferException) {
Log.e(tag, "failed to get protocol.", exception)
}
@ -130,11 +95,6 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
// 检查相机权限
PermissionHelper.checkAndRequestCameraPermissions(this)
// 提示设置服务器
if (!isSet) {
showToastMessage("Server Unset!")
}
}
override fun onResume() {
@ -157,11 +117,6 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
converter.close()
previewDisplayView.visibility = View.GONE
if (isSet) {
sender.close()
isSet = false
}
}
override fun onRequestPermissionsResult(
@ -173,40 +128,6 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults)
}
override fun onDialogPositiveClick(dialog: DialogFragment) {
Log.i(tag, "Confirm Button Clicked.")
val ipEditText = dialog.dialog?.findViewById<EditText>(R.id.targetIP)
val portEditText = dialog.dialog?.findViewById<EditText>(R.id.targetPort)
if (ipEditText == null || portEditText == null) {
Log.e(tag, "Get EditText Failed.")
return
}
val ipInput = ipEditText.text.toString()
val portInput = portEditText.text.toString()
try {
val port = portInput.toInt()
if (isSet) {
sender.close()
isSet = false
}
sender = UdpSender(ipInput, port)
isSet = true
showToastMessage("Server $ipInput:$portInput set successfully! ")
} catch (e: java.lang.NumberFormatException) {
Log.e(tag, "Input error: $e")
}
}
override fun onDialogNegativeClick(dialog: DialogFragment) {
Log.i(tag, "Cancel Button Clicked.")
}
/**
* 启动摄像机
*/
@ -327,38 +248,4 @@ class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListene
}
)
}
private fun showPortSetDialog() {
val dialogFragment = SendDialogFragment()
dialogFragment.show(supportFragmentManager, "SetIpPortDialog")
}
private fun showToastMessage(message: String) {
val duration = Toast.LENGTH_SHORT
Toast.makeText(this, message, duration).show()
}
/**
* 格式化输出捕捉的标志位置
*/
private fun getPoseLandmarksDebugString(poseLandmarks: NormalizedLandmarkList): String {
val debugString = StringBuilder("Pose LandMarks: " + poseLandmarks.landmarkCount + "\n")
for ((landMarkIndex, landmark) in poseLandmarks.landmarkList.withIndex()) {
debugString.append(
"\tLandMark ["
+ landMarkIndex
+ "]: ("
+ landmark.x
+ ","
+ landmark.y
+ ","
+ landmark.z
+ ")\n"
)
}
return debugString.toString()
}
}

View File

@ -1,46 +0,0 @@
package top.rrricardo.motioncapture
import android.app.AlertDialog
import android.app.Dialog
import android.content.Context
import android.os.Bundle
import androidx.fragment.app.DialogFragment
class SendDialogFragment : DialogFragment() {
// 用于将点击事件传递回宿主Activity的接口
interface NoticeDialogListener {
fun onDialogPositiveClick(dialog: DialogFragment)
fun onDialogNegativeClick(dialog: DialogFragment)
}
internal lateinit var listener: NoticeDialogListener
override fun onCreateDialog(savedInstanceState: Bundle?): Dialog {
return activity?.let {
val builder = AlertDialog.Builder(it)
val inflater = requireActivity().layoutInflater
builder.setView(inflater.inflate(R.layout.dialog_set_port, null))
.setPositiveButton(R.string.confirm_button
) { _, _ -> listener.onDialogPositiveClick(this) }
.setNegativeButton(R.string.cancel_button
) { _, _ -> listener.onDialogNegativeClick(this) }
builder.create()
}?: throw java.lang.IllegalStateException("Activity cannot be null")
}
override fun onAttach(context: Context) {
super.onAttach(context)
// 确认宿主Activity实现了点击事件接口
try {
listener = context as NoticeDialogListener
} catch (e: java.lang.ClassCastException) {
// 没有实现
throw java.lang.ClassCastException("$context must implement NoticeDialogListener.")
}
}
}

View File

@ -1,34 +0,0 @@
package top.rrricardo.motioncapture
import android.util.Log
import java.net.DatagramPacket
import java.net.DatagramSocket
import java.net.InetAddress
class UdpSender(val targetIp: String, val targetPort: Int) {
private val datagramSocket = DatagramSocket()
private val tag = "UdpSender"
private var closed = false
fun sendMessage(data: ByteArray) {
if (closed) {
Log.e(tag, "Udp sender has been closed!")
return
}
val packet = DatagramPacket(data, data.size, InetAddress.getByName(targetIp), targetPort)
try {
datagramSocket.send(packet)
} catch (e: java.lang.Exception) {
e.printStackTrace()
Log.e(tag, "UDP send error: $e")
}
}
fun close() {
datagramSocket.close();
closed = true
}
}

View File

@ -1,74 +0,0 @@
package top.rrricardo.motioncapture.models
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList
import java.nio.ByteBuffer
import java.nio.ByteOrder
class PoseLandmark(
private val Type: Int,
private val X: Float,
private val Y: Float,
private val Z: Float,
private val Visibility: Float,
private val TimeStamp: Long) {
fun toByteArray(): ByteArray {
val result = ByteBuffer.allocate(packetLength)
result.order(ByteOrder.LITTLE_ENDIAN)
result.putInt(Type)
result.putFloat(X)
result.putFloat(Y)
result.putFloat(Z)
result.putFloat(Visibility)
result.putLong(TimeStamp)
return result.array()
}
companion object {
const val packetLength = 28
fun valueOf(data: ByteArray): PoseLandmark {
val buffer = ByteBuffer.wrap(data)
buffer.order(ByteOrder.LITTLE_ENDIAN)
val type = buffer.int
val x = buffer.float
val y = buffer.float
val z = buffer.float
val visibility = buffer.float
val timeStamp = buffer.long
return PoseLandmark(type, x, y, z, visibility, timeStamp)
}
fun valueOf(poseLandmarks: NormalizedLandmarkList, timeStamp: Long): Collection<PoseLandmark> {
val result = mutableSetOf<PoseLandmark>()
for ((landmarkIndex, landmark) in poseLandmarks.landmarkList.withIndex()) {
val poseLandmark = PoseLandmark(
landmarkIndex,
landmark.x,
landmark.y,
landmark.z,
landmark.visibility,
timeStamp
)
result.add(poseLandmark)
}
return result
}
}
override fun toString(): String {
val builder = StringBuilder()
builder.append("Time: $TimeStamp, Type: $Type\n")
builder.append("\tX:$X, Y:$Y, Z:$Z, Visibility: $Visibility\n")
return builder.toString()
}
}

View File

@ -1,5 +0,0 @@
<vector android:height="24dp" android:tint="#000000"
android:viewportHeight="24" android:viewportWidth="24"
android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
<path android:fillColor="@android:color/white" android:pathData="M12,5c-3.87,0 -7,3.13 -7,7h2c0,-2.76 2.24,-5 5,-5s5,2.24 5,5h2c0,-3.87 -3.13,-7 -7,-7zM13,14.29c0.88,-0.39 1.5,-1.26 1.5,-2.29 0,-1.38 -1.12,-2.5 -2.5,-2.5S9.5,10.62 9.5,12c0,1.02 0.62,1.9 1.5,2.29v3.3L7.59,21 9,22.41l3,-3 3,3L16.41,21 13,17.59v-3.3zM12,1C5.93,1 1,5.93 1,12h2c0,-4.97 4.03,-9 9,-9s9,4.03 9,9h2c0,-6.07 -4.93,-11 -11,-11z"/>
</vector>

View File

@ -1,34 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:orientation="vertical">
<TextView
android:layout_width="match_parent"
android:layout_height="64dp"
android:text="@string/input_ip_port_hint"
android:gravity="center">
</TextView>
<EditText
android:id="@+id/targetIP"
android:inputType="text"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_margin="20dp"
android:hint="@string/input_ip_hint"
android:autofillHints="IP Address">
</EditText>
<EditText
android:id="@+id/targetPort"
android:inputType="number"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_margin="20dp"
android:hint="@string/input_port_hint"
android:autofillHints="Port Number">
</EditText>
</LinearLayout>

View File

@ -8,11 +8,4 @@
android:icon="@drawable/camera_flip"
android:title="@string/action_flip_camera_name">
</item>
<item
android:id="@+id/action_show_SetIpPortDialog"
app:showAsAction="ifRoom"
android:icon="@drawable/baseline_settings_input_antenna"
android:title="@string/input_ip_port_hint">
</item>
</menu>

View File

@ -2,9 +2,4 @@
<string name="app_name">MotionCapture</string>
<string name="no_camera_access_hint">Please Make Sure the Camera Permission is Given!</string>
<string name="action_flip_camera_name">Flip Camera</string>
<string name="input_ip_port_hint">Input Target IP and Port!</string>
<string name="input_ip_hint">Input IP Address</string>
<string name="input_port_hint">Input Port Number</string>
<string name="confirm_button">Confirm</string>
<string name="cancel_button">Cancel</string>
</resources>

View File

@ -1,6 +1,6 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
plugins {
id 'com.android.application' version '7.4.1' apply false
id 'com.android.library' version '7.4.1' apply false
id 'org.jetbrains.kotlin.android' version '1.8.0' apply false
id 'com.android.application' version '7.3.1' apply false
id 'com.android.library' version '7.3.1' apply false
id 'org.jetbrains.kotlin.android' version '1.7.20' apply false
}

View File

@ -1,6 +1,6 @@
#Sat Jan 07 16:28:48 CST 2023
distributionBase=GRADLE_USER_HOME
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME