Merge pull request 'ParserTest' (#10) from ParserTest into master

Reviewed-on: PostGuard/Canon#10

Canon.Core/Enums/SemanticEnums.cs

@@ -68,3 +68,10 @@ public enum OperatorType
     Or,
     Assign
 }
+
+public enum NumberType
+{
+    Integer,
+    Real,
+    Hex
+}

Canon.Core/LexicalParser/SemanticToken.cs

@@ -4,6 +4,8 @@ namespace Canon.Core.LexicalParser;
 
 using Enums;
 
+using System.Text;
+
 /// <summary>
 /// 词法记号基类
 /// </summary>
@@ -198,13 +200,69 @@ public class OperatorSemanticToken : SemanticToken
 /// <summary>
 /// 数值类型记号
 /// </summary>
+/// TODO：进制表示（只有$1的十六进制表示）
 public class NumberSemanticToken : SemanticToken
 {
     public override SemanticTokenType TokenType => SemanticTokenType.Number;
 
+    public required NumberType NumberType { get; init; }
+    public double Value { get; private init; }
+
     public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
         out NumberSemanticToken? token)
     {
+        StringBuilder buffer = new();
+
+        bool hasDecimalPoint = false;
+        bool hasExponent = false;
+        bool hasMinusSign = false;
+
+        while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E' || now.Value == '-' || now.Value == '+'))
+        {
+            if (now.Value == '.')
+            {
+                if (hasDecimalPoint)
+                {
+                    break;
+                }
+                hasDecimalPoint = true;
+            }
+
+            if (now.Value == 'e' || now.Value == 'E')
+            {
+                if (hasExponent)
+                {
+                    break;
+                }
+                hasExponent = true;
+            }
+
+            if (now.Value == '-' || now.Value == '+')
+            {
+                if (hasMinusSign)
+                {
+                    break;
+                }
+                hasMinusSign = true;
+            }
+
+            buffer.Append(now.Value);
+            now = now.Next;
+        }
+
+        if (double.TryParse(buffer.ToString(), out double value))
+        {
+            token = new NumberSemanticToken
+            {
+                LinePos = linePos,
+                CharacterPos = characterPos,
+                LiteralValue = buffer.ToString(),
+                Value = value,
+                NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer
+            };
+            return true;
+        }
+
         token = null;
         return false;
     }

Canon.Tests/LexicalParserTests/Array.cs

@@ -0,0 +1,6 @@
+namespace Canon.Tests.LexicalParserTests;
+
+public class Array
+{
+    
+}

Canon.Tests/LexicalParserTests/IndentifierTests.cs

@@ -0,0 +1,37 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+using Xunit;
+
+namespace Canon.Tests.LexicalParserTests
+{
+    public class IdentifierTests
+    {
+        [Theory]
+        [InlineData("identifier", true)]
+        [InlineData("_identifier", true)]
+        [InlineData("identifier123", true)]
+        [InlineData("123identifier", false)]
+        [InlineData("identifier_with_underscores", true)]
+        [InlineData("IdentifierWithCamelCase", true)]
+        [InlineData("identifier-with-hyphen", false)]
+        [InlineData("identifier with spaces", false)]
+        [InlineData("identifier_with_special_chars@#", false)]
+        [InlineData("", false)]
+        [InlineData(" ", false)]
+        [InlineData("andand",false)]
+        public void TestParseIdentifier(string input, bool expectedResult)
+        {
+            LinkedList<char> content = Utils.GetLinkedList(input);
+            Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!,
+                out IdentifierSemanticToken? token));
+            if (expectedResult)
+            {
+                Assert.NotNull(token);
+            }
+            else
+            {
+                Assert.Null(token);
+            }
+        }
+    }
+}

Canon.Tests/LexicalParserTests/KeywordTypeTests.cs

@@ -0,0 +1,33 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests;
+
+public class KeywordTypeTests
+{
+    [Theory]
+    [InlineData("program", KeywordType.Program)]
+    [InlineData("const", KeywordType.Const)]
+    [InlineData("var", KeywordType.Var)]
+    [InlineData("procedure", KeywordType.Procedure)]
+    [InlineData("function", KeywordType.Function)]
+    [InlineData("begin", KeywordType.Begin)]
+    [InlineData("end", KeywordType.End)]
+    [InlineData("array", KeywordType.Array)]
+    [InlineData("of", KeywordType.Of)]
+    [InlineData("if", KeywordType.If)]
+    [InlineData("then", KeywordType.Then)]
+    [InlineData("else", KeywordType.Else)]
+    [InlineData("for", KeywordType.For)]
+    [InlineData("to", KeywordType.To)]
+    [InlineData("do", KeywordType.Do)]
+    public void SmokeTest(string input, KeywordType type)
+    {
+        LinkedList<char> content = Utils.GetLinkedList(input);
+
+        Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!,
+            out KeywordSemanticToken? token));
+        Assert.NotNull(token);
+        Assert.Equal(type, token.KeywordType);
+    }
+}

Canon.Tests/LexicalParserTests/NumberTests.cs

@@ -0,0 +1,45 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests
+{
+    public class NumberTests
+    {
+        [Theory]
+        [InlineData("123", 123, NumberType.Integer)]
+        [InlineData("0", 0, NumberType.Integer)]
+        [InlineData("-123", -123, NumberType.Integer)]
+        [InlineData("1.23", 1.23, NumberType.Real)]
+        [InlineData("-1.23", -1.23, NumberType.Real)]
+        [InlineData("0.0", 0.0, NumberType.Real)]
+        [InlineData("1e7", 1e7, NumberType.Real)]
+        [InlineData("1E7", 1E7, NumberType.Real)]
+        [InlineData("1.23e-7", 1.23e-7, NumberType.Real)]
+        [InlineData("1.23E-7", 1.23E-7, NumberType.Real)]
+        [InlineData("1234567890", 1234567890, NumberType.Integer)]
+        [InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)]
+        [InlineData("-1234567890", -1234567890, NumberType.Integer)]
+        [InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)]
+        [InlineData("1e-7", 1e-7, NumberType.Real)]
+        [InlineData("1E-7", 1E-7, NumberType.Real)]
+        [InlineData("1E", 0, NumberType.Real, false)]
+        [InlineData("abc", 0, NumberType.Integer, false)]
+        [InlineData("123abc", 123, NumberType.Integer, true)]
+        public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true)
+        {
+            LinkedList<char> content = Utils.GetLinkedList(input);
+            Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!,
+                out NumberSemanticToken? token));
+            if (expectedResult)
+            {
+                Assert.NotNull(token);
+                Assert.Equal(expected, token.Value);
+                Assert.Equal(expectedNumberType, token.NumberType);
+            }
+            else
+            {
+                Assert.Null(token);
+            }
+        }
+    }
+}

Canon.Tests/LexicalParserTests/OperatorTypeTests.cs

@@ -0,0 +1,50 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests;
+
+public class OperatorTypeTests
+{
+    [Theory]
+    [InlineData("+ 123", OperatorType.Plus, 0u, 0u, true)]
+    [InlineData("1 + 123", OperatorType.Plus, 0u, 2u, true)]
+    [InlineData("+123", OperatorType.Plus, 0u, 0u, true)]
+    [InlineData("m +123", OperatorType.Plus, 0u, 2u, true)]
+    [InlineData("-123", OperatorType.Minus, 0u, 0u, true)]
+    [InlineData("*123", OperatorType.Multiply, 0u, 0u, true)]
+    [InlineData("/123", OperatorType.Divide, 0u, 0u, true)]
+    [InlineData("=123", OperatorType.Equal, 0u, 0u, true)]
+    [InlineData("<123", OperatorType.Less, 0u, 0u, true)]
+    [InlineData(">123", OperatorType.Greater, 0u, 0u, true)]
+    [InlineData("<=123", OperatorType.LessEqual, 0u, 0u, true)]
+    [InlineData(">=123", OperatorType.GreaterEqual, 0u, 0u, true)]
+    [InlineData("<>123", OperatorType.NotEqual, 0u, 0u, true)]
+    [InlineData(":=123", OperatorType.Assign, 0u, 0u, true)]
+    [InlineData("and 123", OperatorType.And, 0u, 0u, true)]
+    [InlineData("or123", OperatorType.Or, 0u, 0u, true)]
+    [InlineData("mod123", OperatorType.Mod, 0u, 0u, true)]
+
+    [InlineData("and123", OperatorType.And, 0u, 0u, false)]
+    [InlineData("andasd", OperatorType.And, 0u, 0u, false)]
+    [InlineData("andand", OperatorType.And, 0u, 0u, false)]
+    [InlineData("<><123", OperatorType.NotEqual, 0u, 0u, false)]
+    [InlineData("<><123", OperatorType.Less, 0u, 0u, false)]
+    [InlineData("<=<123", OperatorType.LessEqual, 0u, 0u, false)]
+    public void SmokeTest(string input, OperatorType type, uint expectedLinePos, uint expectedCharacterPos, bool expectedResult)
+    {
+        LinkedList<char> content = Utils.GetLinkedList(input);
+        Assert.Equal(expectedResult, OperatorSemanticToken.TryParse(expectedLinePos, expectedCharacterPos, content.First!,
+            out OperatorSemanticToken? token));
+        if (expectedResult)
+        {
+            Assert.NotNull(token);
+            Assert.Equal(type, token.OperatorType);
+            Assert.Equal(expectedLinePos, token.LinePos);
+            Assert.Equal(expectedCharacterPos, token.CharacterPos);
+        }
+        else
+        {
+            Assert.Null(token);
+        }
+    }
+}