From 7ea97ea97e29ab275fe7d411f20bda441e4ff749 Mon Sep 17 00:00:00 2001 From: duqoo <92306417+duqoo@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:19:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=AF=B9=E9=83=A8?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E5=9E=8B=E7=9A=84=E6=B5=8B=E8=AF=95=E6=A0=B7?= =?UTF-8?q?=E4=BE=8B=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=B7=BB=E5=8A=A0=E4=BA=86?= =?UTF-8?q?NumberSemanticToken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Canon.Core/Enums/SemanticEnums.cs | 7 +++ Canon.Core/LexicalParser/SemanticToken.cs | 48 ++++++++++++++++++ Canon.Tests/LexicalParserTests/Array.cs | 6 +++ .../LexicalParserTests/IndentifierTests.cs | 37 ++++++++++++++ .../LexicalParserTests/KeywordTypeTests.cs | 33 ++++++++++++ Canon.Tests/LexicalParserTests/NumberTests.cs | 45 +++++++++++++++++ .../LexicalParserTests/OperatorTypeTests.cs | 50 +++++++++++++++++++ 7 files changed, 226 insertions(+) create mode 100644 Canon.Tests/LexicalParserTests/Array.cs create mode 100644 Canon.Tests/LexicalParserTests/IndentifierTests.cs create mode 100644 Canon.Tests/LexicalParserTests/KeywordTypeTests.cs create mode 100644 Canon.Tests/LexicalParserTests/NumberTests.cs create mode 100644 Canon.Tests/LexicalParserTests/OperatorTypeTests.cs diff --git a/Canon.Core/Enums/SemanticEnums.cs b/Canon.Core/Enums/SemanticEnums.cs index fdd9c86..2eeda0d 100644 --- a/Canon.Core/Enums/SemanticEnums.cs +++ b/Canon.Core/Enums/SemanticEnums.cs @@ -68,3 +68,10 @@ public enum OperatorType Or, Assign } + +public enum NumberType +{ + Integer, + Real, + Hex +} diff --git a/Canon.Core/LexicalParser/SemanticToken.cs b/Canon.Core/LexicalParser/SemanticToken.cs index 2e87aea..358c291 100644 --- a/Canon.Core/LexicalParser/SemanticToken.cs +++ b/Canon.Core/LexicalParser/SemanticToken.cs @@ -4,6 +4,8 @@ namespace Canon.Core.LexicalParser; using Enums; +using System.Text; + /// /// 词法记号基类 /// @@ -198,13 +200,59 @@ public class OperatorSemanticToken : SemanticToken /// /// 数值类型记号 /// +/// TODO：进制表示（只有$1的十六进制表示） public class NumberSemanticToken : SemanticToken { public override SemanticTokenType TokenType => SemanticTokenType.Number; + public required NumberType NumberType { get; init; } + public double Value { get; private init; } + public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, out NumberSemanticToken? token) { + StringBuilder buffer = new(); + + bool hasDecimalPoint = false; + bool hasExponent = false; + + while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E')) + { + if (now.Value == '.') + { + if (hasDecimalPoint) + { + break; + } + hasDecimalPoint = true; + } + + if (now.Value == 'e' || now.Value == 'E') + { + if (hasExponent) + { + break; + } + hasExponent = true; + } + + buffer.Append(now.Value); + now = now.Next; + } + + if (double.TryParse(buffer.ToString(), out double value)) + { + token = new NumberSemanticToken + { + LinePos = linePos, + CharacterPos = characterPos, + LiteralValue = buffer.ToString(), + Value = value, + NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer + }; + return true; + } + token = null; return false; } diff --git a/Canon.Tests/LexicalParserTests/Array.cs b/Canon.Tests/LexicalParserTests/Array.cs new file mode 100644 index 0000000..abd8c40 --- /dev/null +++ b/Canon.Tests/LexicalParserTests/Array.cs @@ -0,0 +1,6 @@ +namespace Canon.Tests.LexicalParserTests; + +public class Array +{ + +} diff --git a/Canon.Tests/LexicalParserTests/IndentifierTests.cs b/Canon.Tests/LexicalParserTests/IndentifierTests.cs new file mode 100644 index 0000000..cfb657e --- /dev/null +++ b/Canon.Tests/LexicalParserTests/IndentifierTests.cs @@ -0,0 +1,37 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; +using Xunit; + +namespace Canon.Tests.LexicalParserTests +{ + public class IdentifierTests + { + [Theory] + [InlineData("identifier", true)] + [InlineData("_identifier", true)] + [InlineData("identifier123", true)] + [InlineData("123identifier", false)] + [InlineData("identifier_with_underscores", true)] + [InlineData("IdentifierWithCamelCase", true)] + [InlineData("identifier-with-hyphen", false)] + [InlineData("identifier with spaces", false)] + [InlineData("identifier_with_special_chars@#", false)] + [InlineData("", false)] + [InlineData(" ", false)] + [InlineData("andand",false)] + public void TestParseIdentifier(string input, bool expectedResult) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!, + out IdentifierSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + } + else + { + Assert.Null(token); + } + } + } +} diff --git a/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs new file mode 100644 index 0000000..e36d149 --- /dev/null +++ b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs @@ -0,0 +1,33 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests; + +public class KeywordTypeTests +{ + [Theory] + [InlineData("program", KeywordType.Program)] + [InlineData("const", KeywordType.Const)] + [InlineData("var", KeywordType.Var)] + [InlineData("procedure", KeywordType.Procedure)] + [InlineData("function", KeywordType.Function)] + [InlineData("begin", KeywordType.Begin)] + [InlineData("end", KeywordType.End)] + [InlineData("array", KeywordType.Array)] + [InlineData("of", KeywordType.Of)] + [InlineData("if", KeywordType.If)] + [InlineData("then", KeywordType.Then)] + [InlineData("else", KeywordType.Else)] + [InlineData("for", KeywordType.For)] + [InlineData("to", KeywordType.To)] + [InlineData("do", KeywordType.Do)] + public void SmokeTest(string input, KeywordType type) + { + LinkedList content = Utils.GetLinkedList(input); + + Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!, + out KeywordSemanticToken? token)); + Assert.NotNull(token); + Assert.Equal(type, token.KeywordType); + } +} diff --git a/Canon.Tests/LexicalParserTests/NumberTests.cs b/Canon.Tests/LexicalParserTests/NumberTests.cs new file mode 100644 index 0000000..9e8ab0f --- /dev/null +++ b/Canon.Tests/LexicalParserTests/NumberTests.cs @@ -0,0 +1,45 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests +{ + public class NumberTests + { + [Theory] + [InlineData("123", 123, NumberType.Integer)] + [InlineData("0", 0, NumberType.Integer)] + [InlineData("-123", -123, NumberType.Integer)] + [InlineData("1.23", 1.23, NumberType.Real)] + [InlineData("-1.23", -1.23, NumberType.Real)] + [InlineData("0.0", 0.0, NumberType.Real)] + [InlineData("1e7", 1e7, NumberType.Real)] + [InlineData("1E7", 1E7, NumberType.Real)] + [InlineData("1.23e-7", 1.23e-7, NumberType.Real)] + [InlineData("1.23E-7", 1.23E-7, NumberType.Real)] + [InlineData("1234567890", 1234567890, NumberType.Integer)] + [InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)] + [InlineData("-1234567890", -1234567890, NumberType.Integer)] + [InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)] + [InlineData("1e-7", 1e-7, NumberType.Real)] + [InlineData("1E-7", 1E-7, NumberType.Real)] + [InlineData("1E", 0, NumberType.Real, false)] + [InlineData("abc", 0, NumberType.Integer, false)] + [InlineData("123abc", 0, NumberType.Integer, false)] + public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!, + out NumberSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + Assert.Equal(expected, token.Value); + Assert.Equal(expectedNumberType, token.NumberType); + } + else + { + Assert.Null(token); + } + } + } +} diff --git a/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs new file mode 100644 index 0000000..b14961f --- /dev/null +++ b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs @@ -0,0 +1,50 @@ +using Canon.Core.Enums; +using Canon.Core.LexicalParser; + +namespace Canon.Tests.LexicalParserTests; + +public class OperatorTypeTests +{ + [Theory] + [InlineData("+ 123", OperatorType.Plus, 0u, 0u, true)] + [InlineData("1 + 123", OperatorType.Plus, 0u, 2u, true)] + [InlineData("+123", OperatorType.Plus, 0u, 0u, true)] + [InlineData("m +123", OperatorType.Plus, 0u, 2u, true)] + [InlineData("-123", OperatorType.Minus, 0u, 0u, true)] + [InlineData("*123", OperatorType.Multiply, 0u, 0u, true)] + [InlineData("/123", OperatorType.Divide, 0u, 0u, true)] + [InlineData("=123", OperatorType.Equal, 0u, 0u, true)] + [InlineData("<123", OperatorType.Less, 0u, 0u, true)] + [InlineData(">123", OperatorType.Greater, 0u, 0u, true)] + [InlineData("<=123", OperatorType.LessEqual, 0u, 0u, true)] + [InlineData(">=123", OperatorType.GreaterEqual, 0u, 0u, true)] + [InlineData("<>123", OperatorType.NotEqual, 0u, 0u, true)] + [InlineData(":=123", OperatorType.Assign, 0u, 0u, true)] + [InlineData("and 123", OperatorType.And, 0u, 0u, true)] + [InlineData("or123", OperatorType.Or, 0u, 0u, true)] + [InlineData("mod123", OperatorType.Mod, 0u, 0u, true)] + + [InlineData("and123", OperatorType.And, 0u, 0u, false)] + [InlineData("andasd", OperatorType.And, 0u, 0u, false)] + [InlineData("andand", OperatorType.And, 0u, 0u, false)] + [InlineData("<><123", OperatorType.NotEqual, 0u, 0u, false)] + [InlineData("<><123", OperatorType.Less, 0u, 0u, false)] + [InlineData("<=<123", OperatorType.LessEqual, 0u, 0u, false)] + public void SmokeTest(string input, OperatorType type, uint expectedLinePos, uint expectedCharacterPos, bool expectedResult) + { + LinkedList content = Utils.GetLinkedList(input); + Assert.Equal(expectedResult, OperatorSemanticToken.TryParse(expectedLinePos, expectedCharacterPos, content.First!, + out OperatorSemanticToken? token)); + if (expectedResult) + { + Assert.NotNull(token); + Assert.Equal(type, token.OperatorType); + Assert.Equal(expectedLinePos, token.LinePos); + Assert.Equal(expectedCharacterPos, token.CharacterPos); + } + else + { + Assert.Null(token); + } + } +}