diff --git a/Canon.Core/Enums/SemanticEnums.cs b/Canon.Core/Enums/SemanticEnums.cs index fdd9c86..2eeda0d 100644 --- a/Canon.Core/Enums/SemanticEnums.cs +++ b/Canon.Core/Enums/SemanticEnums.cs @@ -68,3 +68,10 @@ public enum OperatorType Or, Assign } + +public enum NumberType +{ + Integer, + Real, + Hex +} diff --git a/Canon.Core/LexicalParser/SemanticToken.cs b/Canon.Core/LexicalParser/SemanticToken.cs index 2e87aea..358c291 100644 --- a/Canon.Core/LexicalParser/SemanticToken.cs +++ b/Canon.Core/LexicalParser/SemanticToken.cs @@ -4,6 +4,8 @@ namespace Canon.Core.LexicalParser; using Enums; +using System.Text; + /// /// 词法记号基类 /// @@ -198,13 +200,59 @@ public class OperatorSemanticToken : SemanticToken /// /// 数值类型记号 /// +/// TODO:进制表示(只有$1的十六进制表示) public class NumberSemanticToken : SemanticToken { public override SemanticTokenType TokenType => SemanticTokenType.Number; + public required NumberType NumberType { get; init; } + public double Value { get; private init; } + public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, out NumberSemanticToken? token) { + StringBuilder buffer = new(); + + bool hasDecimalPoint = false; + bool hasExponent = false; + + while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E')) + { + if (now.Value == '.') + { + if (hasDecimalPoint) + { + break; + } + hasDecimalPoint = true; + } + + if (now.Value == 'e' || now.Value == 'E') + { + if (hasExponent) + { + break; + } + hasExponent = true; + } + + buffer.Append(now.Value); + now = now.Next; + } + + if (double.TryParse(buffer.ToString(), out double value)) + { + token = new NumberSemanticToken + { + LinePos = linePos, + CharacterPos = characterPos, + LiteralValue = buffer.ToString(), + Value = value, + NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer + }; + return true; + } + token = null; return false; } diff --git a/Canon.Tests/LexicalParserTests/Array.cs b/Canon.Tests/LexicalParserTests/Array.cs new file mode 100644 index 0000000..abd8c40 --- /dev/null +++ b/Canon.Tests/LexicalParserTests/Array.cs @@ -0,0 +1,6 @@ +namespace Canon.Tests.LexicalParserTests; + +public class Array +{ + +} diff --git a/Canon.Tests/LexicalParserTests/IndentifierTests.cs b/Canon.Tests/LexicalParserTests/IndentifierTests.cs new file mode 100644 index 0000000..cfb657e --- /dev/null +++ b/Canon.Tests/LexicalParserTests/IndentifierTests.cs @@ -0,0 +1,37 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; +using Xunit; + +namespace Canon.Tests.LexicalParserTests +{ + public class IdentifierTests + { + [Theory] + [InlineData("identifier", true)] + [InlineData("_identifier", true)] + [InlineData("identifier123", true)] + [InlineData("123identifier", false)] + [InlineData("identifier_with_underscores", true)] + [InlineData("IdentifierWithCamelCase", true)] + [InlineData("identifier-with-hyphen", false)] + [InlineData("identifier with spaces", false)] + [InlineData("identifier_with_special_chars@#", false)] + [InlineData("", false)] + [InlineData(" ", false)] + [InlineData("andand",false)] + public void TestParseIdentifier(string input, bool expectedResult) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!, + out IdentifierSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + } + else + { + Assert.Null(token); + } + } + } +} diff --git a/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs new file mode 100644 index 0000000..e36d149 --- /dev/null +++ b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs @@ -0,0 +1,33 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests; + +public class KeywordTypeTests +{ + [Theory] + [InlineData("program", KeywordType.Program)] + [InlineData("const", KeywordType.Const)] + [InlineData("var", KeywordType.Var)] + [InlineData("procedure", KeywordType.Procedure)] + [InlineData("function", KeywordType.Function)] + [InlineData("begin", KeywordType.Begin)] + [InlineData("end", KeywordType.End)] + [InlineData("array", KeywordType.Array)] + [InlineData("of", KeywordType.Of)] + [InlineData("if", KeywordType.If)] + [InlineData("then", KeywordType.Then)] + [InlineData("else", KeywordType.Else)] + [InlineData("for", KeywordType.For)] + [InlineData("to", KeywordType.To)] + [InlineData("do", KeywordType.Do)] + public void SmokeTest(string input, KeywordType type) + { + LinkedList content = Utils.GetLinkedList(input); + + Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!, + out KeywordSemanticToken? token)); + Assert.NotNull(token); + Assert.Equal(type, token.KeywordType); + } +} diff --git a/Canon.Tests/LexicalParserTests/NumberTests.cs b/Canon.Tests/LexicalParserTests/NumberTests.cs new file mode 100644 index 0000000..9e8ab0f --- /dev/null +++ b/Canon.Tests/LexicalParserTests/NumberTests.cs @@ -0,0 +1,45 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests +{ + public class NumberTests + { + [Theory] + [InlineData("123", 123, NumberType.Integer)] + [InlineData("0", 0, NumberType.Integer)] + [InlineData("-123", -123, NumberType.Integer)] + [InlineData("1.23", 1.23, NumberType.Real)] + [InlineData("-1.23", -1.23, NumberType.Real)] + [InlineData("0.0", 0.0, NumberType.Real)] + [InlineData("1e7", 1e7, NumberType.Real)] + [InlineData("1E7", 1E7, NumberType.Real)] + [InlineData("1.23e-7", 1.23e-7, NumberType.Real)] + [InlineData("1.23E-7", 1.23E-7, NumberType.Real)] + [InlineData("1234567890", 1234567890, NumberType.Integer)] + [InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)] + [InlineData("-1234567890", -1234567890, NumberType.Integer)] + [InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)] + [InlineData("1e-7", 1e-7, NumberType.Real)] + [InlineData("1E-7", 1E-7, NumberType.Real)] + [InlineData("1E", 0, NumberType.Real, false)] + [InlineData("abc", 0, NumberType.Integer, false)] + [InlineData("123abc", 0, NumberType.Integer, false)] + public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!, + out NumberSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + Assert.Equal(expected, token.Value); + Assert.Equal(expectedNumberType, token.NumberType); + } + else + { + Assert.Null(token); + } + } + } +} diff --git a/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs new file mode 100644 index 0000000..b14961f --- /dev/null +++ b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs @@ -0,0 +1,50 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests; + +public class OperatorTypeTests +{ + [Theory] + [InlineData("+ 123", OperatorType.Plus, 0u, 0u, true)] + [InlineData("1 + 123", OperatorType.Plus, 0u, 2u, true)] + [InlineData("+123", OperatorType.Plus, 0u, 0u, true)] + [InlineData("m +123", OperatorType.Plus, 0u, 2u, true)] + [InlineData("-123", OperatorType.Minus, 0u, 0u, true)] + [InlineData("*123", OperatorType.Multiply, 0u, 0u, true)] + [InlineData("/123", OperatorType.Divide, 0u, 0u, true)] + [InlineData("=123", OperatorType.Equal, 0u, 0u, true)] + [InlineData("<123", OperatorType.Less, 0u, 0u, true)] + [InlineData(">123", OperatorType.Greater, 0u, 0u, true)] + [InlineData("<=123", OperatorType.LessEqual, 0u, 0u, true)] + [InlineData(">=123", OperatorType.GreaterEqual, 0u, 0u, true)] + [InlineData("<>123", OperatorType.NotEqual, 0u, 0u, true)] + [InlineData(":=123", OperatorType.Assign, 0u, 0u, true)] + [InlineData("and 123", OperatorType.And, 0u, 0u, true)] + [InlineData("or123", OperatorType.Or, 0u, 0u, true)] + [InlineData("mod123", OperatorType.Mod, 0u, 0u, true)] + + [InlineData("and123", OperatorType.And, 0u, 0u, false)] + [InlineData("andasd", OperatorType.And, 0u, 0u, false)] + [InlineData("andand", OperatorType.And, 0u, 0u, false)] + [InlineData("<><123", OperatorType.NotEqual, 0u, 0u, false)] + [InlineData("<><123", OperatorType.Less, 0u, 0u, false)] + [InlineData("<=<123", OperatorType.LessEqual, 0u, 0u, false)] + public void SmokeTest(string input, OperatorType type, uint expectedLinePos, uint expectedCharacterPos, bool expectedResult) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, OperatorSemanticToken.TryParse(expectedLinePos, expectedCharacterPos, content.First!, + out OperatorSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + Assert.Equal(type, token.OperatorType); + Assert.Equal(expectedLinePos, token.LinePos); + Assert.Equal(expectedCharacterPos, token.CharacterPos); + } + else + { + Assert.Null(token); + } + } +}