feat: 词法分析器的基本功能(#12)

Co-authored-by: jackfiled <xcrenchangjun@outlook.com>
Reviewed-on: PostGuard/Canon#12
Co-authored-by: Huaps <1183155719@qq.com>
Co-committed-by: Huaps <1183155719@qq.com>

Canon.Core/Enums/SemanticEnums.cs

@@ -8,14 +8,12 @@ public enum SemanticTokenType
     Delimiter,
     Identifier,
     Character,
+    Empty,
+    Error,  // 加了一个错误token
     /// <summary>
     /// 语法分析中的栈底符号
     /// </summary>
-    End,
-    /// <summary>
-    /// 语法分析中的空串符号
-    /// </summary>
-    Empty
+    End
 }
 
 public enum DelimiterType
@@ -89,3 +87,11 @@ public enum NumberType
     Real,
     Hex
 }
+
+public enum StateType
+{
+    Word,
+    Digit,
+    Delimiter,
+    Other
+}