feature: Update gradle plugin to 7.4.1

修复IP地址输入框无法输入符号问题
将时间戳单位修改为毫秒

.idea/deploymentTargetDropDown.xml

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="deploymentTargetDropDown">
-    <targetSelectedWithDropDown>
+    <runningDeviceTargetSelectedWithDropDown>
       <Target>
-        <type value="QUICK_BOOT_TARGET" />
+        <type value="RUNNING_DEVICE_TARGET" />
         <deviceKey>
           <Key>
-            <type value="VIRTUAL_DEVICE_PATH" />
+            <type value="SERIAL_NUMBER" />
-            <value value="$USER_HOME$/.android/avd/Pixel_3a_API_33_x86_64.avd" />
+            <value value="adb-3040223527000PT-yH8BGJ._adb-tls-connect._tcp" />
           </Key>
         </deviceKey>
       </Target>
-    </targetSelectedWithDropDown>
+    </runningDeviceTargetSelectedWithDropDown>
-    <timeTargetWasSelectedWithDropDown value="2023-01-10T13:25:57.409214Z" />
+    <timeTargetWasSelectedWithDropDown value="2023-01-17T14:13:43.027203Z" />
   </component>
 </project>

.idea/gradle.xml

@@ -7,6 +7,7 @@
         <option name="testRunner" value="GRADLE" />
         <option name="distributionType" value="DEFAULT_WRAPPED" />
         <option name="externalProjectPath" value="$PROJECT_DIR$" />
+        <option name="gradleJvm" value="Embedded JDK" />
         <option name="modules">
           <set>
             <option value="$PROJECT_DIR$" />

.idea/misc.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ExternalStorageConfigurationManager" enabled="true" />
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="Android Studio default JDK" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/build/classes" />
   </component>
   <component name="ProjectType">

CHANGELOG.md

@@ -0,0 +1,21 @@
+# CHANGELOG
+
+### 2023-1-7
+
+-  创建git仓库
+
+- 参考[GitHub - LeeJeongHwi/Mediapipe_pose_Tracking_AAR_example: Mediapipe Pose Tracking AAR (android) Example](https://github.com/LeeJeongHwi/Mediapipe_pose_Tracking_AAR_example)，使用`Kotlin`开始开发。
+
+- 基本实现动作捕捉的问题。
+
+### 2023-1-11
+
+- 参考[how to switch cameras on Android? · Issue #1842 · google/mediapipe · GitHub](https://github.com/google/mediapipe/issues/1842)添加翻转摄像头功能。
+
+### 2023-1-17
+
+- 自行编译`MediaPipe`框架，解决获取捕捉数据的问题。
+
+### 2023-1-19
+
+- 参考[Pose landmarks: segmentation seems to be enabled by default · Issue #2454 · google/mediapipe · GitHub](https://github.com/google/mediapipe/issues/2454)，解决自带人体遮罩的问题。

README.md

@@ -0,0 +1,23 @@
+# MotionCaptureAndroid
+
+动作捕捉系统数据采集Android应用程序
+
+基于Google开源机器学习方案[MediaPipe](https://google.github.io/mediapipe/)中的`Pose`解决方案开发。
+
+## 开发
+
+开发使用的IDE：
+
+- [Android Studio](https://developer.android.google.cn/studio/)
+
+使用`Android Studio`导入项目即可。
+
+其中编译`MediaPipe`相关`AAR`组件的步骤详见[编译MediaPipe框架 - Ricardo的博客](https://rrricardo.top/blog/2022/11/11/compile-mediapipe/)。
+
+### TODO
+
+- ~~MediaPipe相关功能实现~~
+
+- 局域网里主机的发现
+
+- 同主机之间的高效数据传输

app/build.gradle

@@ -34,13 +34,15 @@ android {
 
 dependencies {
     // Android 依赖库
-    implementation 'androidx.core:core-ktx:1.7.0'
+    implementation 'androidx.core:core-ktx:1.9.0'
-    implementation 'androidx.appcompat:appcompat:1.4.1'
+    implementation 'androidx.appcompat:appcompat:1.5.1'
-    implementation 'com.google.android.material:material:1.5.0'
+    implementation 'com.google.android.material:material:1.7.0'
-    implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+    implementation 'androidx.core:core-ktx:latest.release'
+    implementation 'androidx.core:core-ktx:+'
     testImplementation 'junit:junit:4.13.2'
     androidTestImplementation 'androidx.test.ext:junit:1.1.3'
-    androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0'
 
     // MediaPipe依赖文件
     implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
@@ -49,7 +51,7 @@ dependencies {
     implementation 'com.google.flogger:flogger-system-backend:latest.release'
     implementation 'com.google.code.findbugs:jsr305:latest.release'
     implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-javalite:3.19.1'
+    implementation 'com.google.protobuf:protobuf-javalite:latest.release'
     // CameraX core library
     def camerax_version = "1.2.0"
     implementation "androidx.camera:camera-core:$camerax_version"

app/src/main/AndroidManifest.xml

@@ -3,6 +3,8 @@
     xmlns:tools="http://schemas.android.com/tools">
     <!-- For using the Camera -->
     <uses-permission android:name="android.permission.CAMERA" />
+    <!-- For sending data -->
+    <uses-permission android:name="android.permission.INTERNET"/>
 
     <uses-feature android:name="android.hardware.camera" />
     <!-- For mediapipe -->

app/src/main/java/top/rrricardo/motioncapture/MainActivity.kt

@@ -6,7 +6,10 @@ import android.os.Bundle
 import android.util.Log
 import android.util.Size
 import android.view.*
+import android.widget.EditText
+import android.widget.Toast
 import androidx.appcompat.widget.Toolbar
+import androidx.fragment.app.DialogFragment
 import com.google.mediapipe.components.CameraHelper.CameraFacing
 import com.google.mediapipe.components.CameraXPreviewHelper
 import com.google.mediapipe.components.ExternalTextureConverter
@@ -15,10 +18,13 @@ import com.google.mediapipe.components.PermissionHelper
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList
 import com.google.mediapipe.framework.AndroidAssetUtil
 import com.google.mediapipe.framework.PacketGetter
+import com.google.mediapipe.framework.ProtoUtil
 import com.google.mediapipe.glutil.EglManager
 import com.google.protobuf.InvalidProtocolBufferException
+import top.rrricardo.motioncapture.models.PoseLandmark
+import java.nio.ByteBuffer
 
-class MainActivity : AppCompatActivity() {
+class MainActivity : AppCompatActivity(), SendDialogFragment.NoticeDialogListener {
     // 一些在设置MediaPipe时会用到的字符串常量
     private val tag = "MainActivity"
     private val binaryGraphName = "pose_tracking_gpu.binarypb"
@@ -33,13 +39,14 @@ class MainActivity : AppCompatActivity() {
     lateinit var converter: ExternalTextureConverter
     lateinit var cameraHelper: CameraXPreviewHelper
     private var cameraFacing = CameraFacing.BACK
+    private var isSet = false
+    private lateinit var sender: UdpSender
 
     private lateinit var toolbar: Toolbar
 
     init {
         // 加载项目中用到的jni库
         System.loadLibrary("mediapipe_jni")
-        System.loadLibrary("opencv_java3")
     }
 
     override fun onCreate(savedInstanceState: Bundle?) {
@@ -55,7 +62,13 @@ class MainActivity : AppCompatActivity() {
                     Log.i(tag, "Flip Camera Button Pressed.")
                     flipCamera()
                     true
-                } else -> {
+                }
+                R.id.action_show_SetIpPortDialog -> {
+                    Log.i(tag, "show SetIpPortDialog Button Pressed.")
+                    showPortSetDialog()
+                    true
+                }
+                else -> {
                     super.onMenuItemSelected(it.itemId, it)
                 }
             }
@@ -79,15 +92,37 @@ class MainActivity : AppCompatActivity() {
 
         processor.videoSurfaceOutput.setFlipY(true)
 
-
+        // 捕获获得的坐标数据包
-        // 展示
+        // 遇到InvalidProtocolBufferException
+        // 参考 https://github.com/google/mediapipe/issues/1013
+        ProtoUtil.registerTypeName(NormalizedLandmarkList::class.java,
+            "mediapipe.NormalizedLandmarkList")
         processor.addPacketCallback(
             outputLandmarksStreamName
         ) {
+            Log.i(tag, "Received Landmark Packets.")
             try {
-                val landmarkRaw = PacketGetter.getProtoBytes(it)
+                Log.i(tag, "Receive Pose Packet.")
-                val poseLandmarks = NormalizedLandmarkList.parseFrom(landmarkRaw)
+                // 无法采用这种方法获取packet
-                // 从这里可以获得每个特征点的座标
+                // 回报Fatal signal 6 (SIGABRT), code -1 (SI_QUEUE) in tid 15943 (Thread-9), pid 15810 (o.motioncapture)
+                /*val packetRaw = PacketGetter.getBytes(it)
+                val landmarks = NormalizedLandmarkList.parseFrom(packetRaw)
+                Log.i(tag, getPoseLandmarksDebugString(landmarks))*/
+                val landmarks = PacketGetter.getProto(it,
+                    NormalizedLandmarkList.getDefaultInstance())
+                val poseLandmarks = PoseLandmark.valueOf(landmarks, System.currentTimeMillis())
+
+                val buffer = ByteBuffer.allocate(
+                    poseLandmarks.size * PoseLandmark.packetLength)
+
+                for (poseLandmark in poseLandmarks) {
+                    buffer.put(poseLandmark.toByteArray())
+                }
+
+                if (isSet) {
+                    sender.sendMessage(buffer.array())
+                }
+
             } catch (exception: InvalidProtocolBufferException) {
                 Log.e(tag, "failed to get protocol.", exception)
             }
@@ -95,6 +130,11 @@ class MainActivity : AppCompatActivity() {
 
         // 检查相机权限
         PermissionHelper.checkAndRequestCameraPermissions(this)
+
+        // 提示设置服务器
+        if (!isSet) {
+            showToastMessage("Server Unset!")
+        }
     }
 
     override fun onResume() {
@@ -117,6 +157,11 @@ class MainActivity : AppCompatActivity() {
 
         converter.close()
         previewDisplayView.visibility = View.GONE
+
+        if (isSet) {
+            sender.close()
+            isSet = false
+        }
     }
 
     override fun onRequestPermissionsResult(
@@ -128,6 +173,40 @@ class MainActivity : AppCompatActivity() {
         PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults)
     }
 
+    override fun onDialogPositiveClick(dialog: DialogFragment) {
+        Log.i(tag, "Confirm Button Clicked.")
+
+        val ipEditText = dialog.dialog?.findViewById<EditText>(R.id.targetIP)
+        val portEditText = dialog.dialog?.findViewById<EditText>(R.id.targetPort)
+
+        if (ipEditText == null || portEditText == null) {
+            Log.e(tag, "Get EditText Failed.")
+            return
+        }
+
+        val ipInput = ipEditText.text.toString()
+        val portInput = portEditText.text.toString()
+
+        try {
+            val port = portInput.toInt()
+            if (isSet) {
+                sender.close()
+                isSet = false
+            }
+
+            sender = UdpSender(ipInput, port)
+            isSet = true
+
+            showToastMessage("Server $ipInput:$portInput set successfully! ")
+        } catch (e: java.lang.NumberFormatException) {
+            Log.e(tag, "Input error: $e")
+        }
+    }
+
+    override fun onDialogNegativeClick(dialog: DialogFragment) {
+        Log.i(tag, "Cancel Button Clicked.")
+    }
+
     /**
      * 启动摄像机
      */
@@ -248,4 +327,38 @@ class MainActivity : AppCompatActivity() {
             }
         )
     }
+
+    private fun showPortSetDialog() {
+        val dialogFragment = SendDialogFragment()
+        dialogFragment.show(supportFragmentManager, "SetIpPortDialog")
+    }
+
+    private fun showToastMessage(message: String) {
+        val duration = Toast.LENGTH_SHORT
+
+        Toast.makeText(this, message, duration).show()
+    }
+
+
+    /**
+     * 格式化输出捕捉的标志位置
+     */
+    private fun getPoseLandmarksDebugString(poseLandmarks: NormalizedLandmarkList): String {
+        val debugString = StringBuilder("Pose LandMarks: " + poseLandmarks.landmarkCount + "\n")
+        for ((landMarkIndex, landmark) in poseLandmarks.landmarkList.withIndex()) {
+            debugString.append(
+                "\tLandMark ["
+            + landMarkIndex
+            + "]: ("
+            + landmark.x
+            + ","
+            + landmark.y
+            + ","
+            + landmark.z
+            + ")\n"
+            )
+        }
+
+        return debugString.toString()
+    }
 }

app/src/main/java/top/rrricardo/motioncapture/SendDialogFragment.kt

@@ -0,0 +1,46 @@
+package top.rrricardo.motioncapture
+
+import android.app.AlertDialog
+import android.app.Dialog
+import android.content.Context
+import android.os.Bundle
+import androidx.fragment.app.DialogFragment
+
+class SendDialogFragment : DialogFragment() {
+    // 用于将点击事件传递回宿主Activity的接口
+    interface NoticeDialogListener {
+        fun onDialogPositiveClick(dialog: DialogFragment)
+        fun onDialogNegativeClick(dialog: DialogFragment)
+    }
+
+    internal lateinit var listener: NoticeDialogListener
+
+    override fun onCreateDialog(savedInstanceState: Bundle?): Dialog {
+        return activity?.let {
+            val builder = AlertDialog.Builder(it)
+            val inflater = requireActivity().layoutInflater
+
+            builder.setView(inflater.inflate(R.layout.dialog_set_port, null))
+                .setPositiveButton(R.string.confirm_button
+                ) { _, _ -> listener.onDialogPositiveClick(this) }
+                .setNegativeButton(R.string.cancel_button
+                ) { _, _ -> listener.onDialogNegativeClick(this) }
+
+
+
+            builder.create()
+        }?: throw java.lang.IllegalStateException("Activity cannot be null")
+    }
+
+    override fun onAttach(context: Context) {
+        super.onAttach(context)
+
+        // 确认宿主Activity实现了点击事件接口
+        try {
+            listener = context as NoticeDialogListener
+        } catch (e: java.lang.ClassCastException) {
+            // 没有实现
+            throw java.lang.ClassCastException("$context must implement NoticeDialogListener.")
+        }
+    }
+}

app/src/main/java/top/rrricardo/motioncapture/UdpSender.kt

@@ -0,0 +1,34 @@
+package top.rrricardo.motioncapture
+
+import android.util.Log
+import java.net.DatagramPacket
+import java.net.DatagramSocket
+import java.net.InetAddress
+
+class UdpSender(val targetIp: String, val targetPort: Int) {
+    private val datagramSocket = DatagramSocket()
+    private val tag = "UdpSender"
+    private var closed = false
+
+    fun sendMessage(data: ByteArray) {
+        if (closed) {
+            Log.e(tag, "Udp sender has been closed!")
+            return
+        }
+
+        val packet = DatagramPacket(data, data.size, InetAddress.getByName(targetIp), targetPort)
+
+        try {
+            datagramSocket.send(packet)
+        } catch (e: java.lang.Exception) {
+            e.printStackTrace()
+            Log.e(tag, "UDP send error: $e")
+        }
+    }
+
+    fun close() {
+        datagramSocket.close();
+        closed = true
+    }
+
+}

app/src/main/java/top/rrricardo/motioncapture/models/PoseLandmark.kt

@@ -0,0 +1,74 @@
+package top.rrricardo.motioncapture.models
+
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+class PoseLandmark(
+    private val Type: Int,
+    private val X: Float,
+    private val Y: Float,
+    private val Z: Float,
+    private val Visibility: Float,
+    private val TimeStamp: Long) {
+
+    fun toByteArray(): ByteArray {
+        val result = ByteBuffer.allocate(packetLength)
+        result.order(ByteOrder.LITTLE_ENDIAN)
+
+        result.putInt(Type)
+        result.putFloat(X)
+        result.putFloat(Y)
+        result.putFloat(Z)
+        result.putFloat(Visibility)
+        result.putLong(TimeStamp)
+
+        return result.array()
+    }
+
+    companion object {
+        const val packetLength = 28
+
+        fun valueOf(data: ByteArray): PoseLandmark {
+            val buffer = ByteBuffer.wrap(data)
+            buffer.order(ByteOrder.LITTLE_ENDIAN)
+
+            val type = buffer.int
+            val x = buffer.float
+            val y = buffer.float
+            val z = buffer.float
+            val visibility = buffer.float
+            val timeStamp = buffer.long
+
+            return PoseLandmark(type, x, y, z, visibility, timeStamp)
+        }
+
+        fun valueOf(poseLandmarks: NormalizedLandmarkList, timeStamp: Long): Collection<PoseLandmark> {
+            val result = mutableSetOf<PoseLandmark>()
+
+            for ((landmarkIndex, landmark) in poseLandmarks.landmarkList.withIndex()) {
+                val poseLandmark = PoseLandmark(
+                    landmarkIndex,
+                    landmark.x,
+                    landmark.y,
+                    landmark.z,
+                    landmark.visibility,
+                    timeStamp
+                )
+
+                result.add(poseLandmark)
+            }
+
+            return result
+        }
+    }
+
+    override fun toString(): String {
+        val builder = StringBuilder()
+
+        builder.append("Time: $TimeStamp, Type: $Type\n")
+        builder.append("\tX:$X, Y:$Y, Z:$Z, Visibility: $Visibility\n")
+
+        return builder.toString()
+    }
+}

app/src/main/res/drawable/baseline_settings_input_antenna.xml

@@ -0,0 +1,5 @@
+<vector android:height="24dp" android:tint="#000000"
+    android:viewportHeight="24" android:viewportWidth="24"
+    android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
+    <path android:fillColor="@android:color/white" android:pathData="M12,5c-3.87,0 -7,3.13 -7,7h2c0,-2.76 2.24,-5 5,-5s5,2.24 5,5h2c0,-3.87 -3.13,-7 -7,-7zM13,14.29c0.88,-0.39 1.5,-1.26 1.5,-2.29 0,-1.38 -1.12,-2.5 -2.5,-2.5S9.5,10.62 9.5,12c0,1.02 0.62,1.9 1.5,2.29v3.3L7.59,21 9,22.41l3,-3 3,3L16.41,21 13,17.59v-3.3zM12,1C5.93,1 1,5.93 1,12h2c0,-4.97 4.03,-9 9,-9s9,4.03 9,9h2c0,-6.07 -4.93,-11 -11,-11z"/>
+</vector>

app/src/main/res/layout/dialog_set_port.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="wrap_content"
+    android:layout_height="wrap_content"
+    android:orientation="vertical">
+    <TextView
+        android:layout_width="match_parent"
+        android:layout_height="64dp"
+        android:text="@string/input_ip_port_hint"
+        android:gravity="center">
+    </TextView>
+
+    <EditText
+        android:id="@+id/targetIP"
+        android:inputType="text"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_margin="20dp"
+        android:hint="@string/input_ip_hint"
+        android:autofillHints="IP Address">
+    </EditText>
+
+    <EditText
+        android:id="@+id/targetPort"
+        android:inputType="number"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_margin="20dp"
+        android:hint="@string/input_port_hint"
+        android:autofillHints="Port Number">
+
+    </EditText>
+
+</LinearLayout>

app/src/main/res/menu/toolbar_menu.xml

@@ -8,4 +8,11 @@
         android:icon="@drawable/camera_flip"
         android:title="@string/action_flip_camera_name">
     </item>
+
+    <item
+        android:id="@+id/action_show_SetIpPortDialog"
+        app:showAsAction="ifRoom"
+        android:icon="@drawable/baseline_settings_input_antenna"
+        android:title="@string/input_ip_port_hint">
+    </item>
 </menu>

app/src/main/res/values/strings.xml

@@ -2,4 +2,9 @@
     <string name="app_name">MotionCapture</string>
     <string name="no_camera_access_hint">Please Make Sure the Camera Permission is Given!</string>
     <string name="action_flip_camera_name">Flip Camera</string>
+    <string name="input_ip_port_hint">Input Target IP and Port!</string>
+    <string name="input_ip_hint">Input IP Address</string>
+    <string name="input_port_hint">Input Port Number</string>
+    <string name="confirm_button">Confirm</string>
+    <string name="cancel_button">Cancel</string>
 </resources>

build.gradle

@@ -1,6 +1,6 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 plugins {
-    id 'com.android.application' version '7.3.1' apply false
+    id 'com.android.application' version '7.4.1' apply false
-    id 'com.android.library' version '7.3.1' apply false
+    id 'com.android.library' version '7.4.1' apply false
-    id 'org.jetbrains.kotlin.android' version '1.7.20' apply false
+    id 'org.jetbrains.kotlin.android' version '1.8.0' apply false
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 #Sat Jan 07 16:28:48 CST 2023
 distributionBase=GRADLE_USER_HOME
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME