From 7ea97ea97e29ab275fe7d411f20bda441e4ff749 Mon Sep 17 00:00:00 2001
From: duqoo <92306417+duqoo@users.noreply.github.com>
Date: Wed, 13 Mar 2024 16:19:07 +0800
Subject: [PATCH 1/2] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=AF=B9?=
=?UTF-8?q?=E9=83=A8=E5=88=86=E7=B1=BB=E5=9E=8B=E7=9A=84=E6=B5=8B=E8=AF=95?=
=?UTF-8?q?=E6=A0=B7=E4=BE=8B=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=B7=BB=E5=8A=A0?=
=?UTF-8?q?=E4=BA=86NumberSemanticToken?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
Canon.Core/Enums/SemanticEnums.cs | 7 +++
Canon.Core/LexicalParser/SemanticToken.cs | 48 ++++++++++++++++++
Canon.Tests/LexicalParserTests/Array.cs | 6 +++
.../LexicalParserTests/IndentifierTests.cs | 37 ++++++++++++++
.../LexicalParserTests/KeywordTypeTests.cs | 33 ++++++++++++
Canon.Tests/LexicalParserTests/NumberTests.cs | 45 +++++++++++++++++
.../LexicalParserTests/OperatorTypeTests.cs | 50 +++++++++++++++++++
7 files changed, 226 insertions(+)
create mode 100644 Canon.Tests/LexicalParserTests/Array.cs
create mode 100644 Canon.Tests/LexicalParserTests/IndentifierTests.cs
create mode 100644 Canon.Tests/LexicalParserTests/KeywordTypeTests.cs
create mode 100644 Canon.Tests/LexicalParserTests/NumberTests.cs
create mode 100644 Canon.Tests/LexicalParserTests/OperatorTypeTests.cs
diff --git a/Canon.Core/Enums/SemanticEnums.cs b/Canon.Core/Enums/SemanticEnums.cs
index fdd9c86..2eeda0d 100644
--- a/Canon.Core/Enums/SemanticEnums.cs
+++ b/Canon.Core/Enums/SemanticEnums.cs
@@ -68,3 +68,10 @@ public enum OperatorType
Or,
Assign
}
+
+public enum NumberType
+{
+ Integer,
+ Real,
+ Hex
+}
diff --git a/Canon.Core/LexicalParser/SemanticToken.cs b/Canon.Core/LexicalParser/SemanticToken.cs
index 2e87aea..358c291 100644
--- a/Canon.Core/LexicalParser/SemanticToken.cs
+++ b/Canon.Core/LexicalParser/SemanticToken.cs
@@ -4,6 +4,8 @@ namespace Canon.Core.LexicalParser;
using Enums;
+using System.Text;
+
///
/// 词法记号基类
///
@@ -198,13 +200,59 @@ public class OperatorSemanticToken : SemanticToken
///
/// 数值类型记号
///
+/// TODO：进制表示（只有$1的十六进制表示）
public class NumberSemanticToken : SemanticToken
{
public override SemanticTokenType TokenType => SemanticTokenType.Number;
+ public required NumberType NumberType { get; init; }
+ public double Value { get; private init; }
+
public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now,
out NumberSemanticToken? token)
{
+ StringBuilder buffer = new();
+
+ bool hasDecimalPoint = false;
+ bool hasExponent = false;
+
+ while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E'))
+ {
+ if (now.Value == '.')
+ {
+ if (hasDecimalPoint)
+ {
+ break;
+ }
+ hasDecimalPoint = true;
+ }
+
+ if (now.Value == 'e' || now.Value == 'E')
+ {
+ if (hasExponent)
+ {
+ break;
+ }
+ hasExponent = true;
+ }
+
+ buffer.Append(now.Value);
+ now = now.Next;
+ }
+
+ if (double.TryParse(buffer.ToString(), out double value))
+ {
+ token = new NumberSemanticToken
+ {
+ LinePos = linePos,
+ CharacterPos = characterPos,
+ LiteralValue = buffer.ToString(),
+ Value = value,
+ NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer
+ };
+ return true;
+ }
+
token = null;
return false;
}
diff --git a/Canon.Tests/LexicalParserTests/Array.cs b/Canon.Tests/LexicalParserTests/Array.cs
new file mode 100644
index 0000000..abd8c40
--- /dev/null
+++ b/Canon.Tests/LexicalParserTests/Array.cs
@@ -0,0 +1,6 @@
+namespace Canon.Tests.LexicalParserTests;
+
+public class Array
+{
+
+}
diff --git a/Canon.Tests/LexicalParserTests/IndentifierTests.cs b/Canon.Tests/LexicalParserTests/IndentifierTests.cs
new file mode 100644
index 0000000..cfb657e
--- /dev/null
+++ b/Canon.Tests/LexicalParserTests/IndentifierTests.cs
@@ -0,0 +1,37 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+using Xunit;
+
+namespace Canon.Tests.LexicalParserTests
+{
+ public class IdentifierTests
+ {
+ [Theory]
+ [InlineData("identifier", true)]
+ [InlineData("_identifier", true)]
+ [InlineData("identifier123", true)]
+ [InlineData("123identifier", false)]
+ [InlineData("identifier_with_underscores", true)]
+ [InlineData("IdentifierWithCamelCase", true)]
+ [InlineData("identifier-with-hyphen", false)]
+ [InlineData("identifier with spaces", false)]
+ [InlineData("identifier_with_special_chars@#", false)]
+ [InlineData("", false)]
+ [InlineData(" ", false)]
+ [InlineData("andand",false)]
+ public void TestParseIdentifier(string input, bool expectedResult)
+ {
+ LinkedList content = Utils.GetLinkedList(input);
+ Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!,
+ out IdentifierSemanticToken? token));
+ if (expectedResult)
+ {
+ Assert.NotNull(token);
+ }
+ else
+ {
+ Assert.Null(token);
+ }
+ }
+ }
+}
diff --git a/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs
new file mode 100644
index 0000000..e36d149
--- /dev/null
+++ b/Canon.Tests/LexicalParserTests/KeywordTypeTests.cs
@@ -0,0 +1,33 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests;
+
+public class KeywordTypeTests
+{
+ [Theory]
+ [InlineData("program", KeywordType.Program)]
+ [InlineData("const", KeywordType.Const)]
+ [InlineData("var", KeywordType.Var)]
+ [InlineData("procedure", KeywordType.Procedure)]
+ [InlineData("function", KeywordType.Function)]
+ [InlineData("begin", KeywordType.Begin)]
+ [InlineData("end", KeywordType.End)]
+ [InlineData("array", KeywordType.Array)]
+ [InlineData("of", KeywordType.Of)]
+ [InlineData("if", KeywordType.If)]
+ [InlineData("then", KeywordType.Then)]
+ [InlineData("else", KeywordType.Else)]
+ [InlineData("for", KeywordType.For)]
+ [InlineData("to", KeywordType.To)]
+ [InlineData("do", KeywordType.Do)]
+ public void SmokeTest(string input, KeywordType type)
+ {
+ LinkedList content = Utils.GetLinkedList(input);
+
+ Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!,
+ out KeywordSemanticToken? token));
+ Assert.NotNull(token);
+ Assert.Equal(type, token.KeywordType);
+ }
+}
diff --git a/Canon.Tests/LexicalParserTests/NumberTests.cs b/Canon.Tests/LexicalParserTests/NumberTests.cs
new file mode 100644
index 0000000..9e8ab0f
--- /dev/null
+++ b/Canon.Tests/LexicalParserTests/NumberTests.cs
@@ -0,0 +1,45 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests
+{
+ public class NumberTests
+ {
+ [Theory]
+ [InlineData("123", 123, NumberType.Integer)]
+ [InlineData("0", 0, NumberType.Integer)]
+ [InlineData("-123", -123, NumberType.Integer)]
+ [InlineData("1.23", 1.23, NumberType.Real)]
+ [InlineData("-1.23", -1.23, NumberType.Real)]
+ [InlineData("0.0", 0.0, NumberType.Real)]
+ [InlineData("1e7", 1e7, NumberType.Real)]
+ [InlineData("1E7", 1E7, NumberType.Real)]
+ [InlineData("1.23e-7", 1.23e-7, NumberType.Real)]
+ [InlineData("1.23E-7", 1.23E-7, NumberType.Real)]
+ [InlineData("1234567890", 1234567890, NumberType.Integer)]
+ [InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)]
+ [InlineData("-1234567890", -1234567890, NumberType.Integer)]
+ [InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)]
+ [InlineData("1e-7", 1e-7, NumberType.Real)]
+ [InlineData("1E-7", 1E-7, NumberType.Real)]
+ [InlineData("1E", 0, NumberType.Real, false)]
+ [InlineData("abc", 0, NumberType.Integer, false)]
+ [InlineData("123abc", 0, NumberType.Integer, false)]
+ public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true)
+ {
+ LinkedList content = Utils.GetLinkedList(input);
+ Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!,
+ out NumberSemanticToken? token));
+ if (expectedResult)
+ {
+ Assert.NotNull(token);
+ Assert.Equal(expected, token.Value);
+ Assert.Equal(expectedNumberType, token.NumberType);
+ }
+ else
+ {
+ Assert.Null(token);
+ }
+ }
+ }
+}
diff --git a/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs
new file mode 100644
index 0000000..b14961f
--- /dev/null
+++ b/Canon.Tests/LexicalParserTests/OperatorTypeTests.cs
@@ -0,0 +1,50 @@
+using Canon.Core.Enums;
+using Canon.Core.LexicalParser;
+
+namespace Canon.Tests.LexicalParserTests;
+
+public class OperatorTypeTests
+{
+ [Theory]
+ [InlineData("+ 123", OperatorType.Plus, 0u, 0u, true)]
+ [InlineData("1 + 123", OperatorType.Plus, 0u, 2u, true)]
+ [InlineData("+123", OperatorType.Plus, 0u, 0u, true)]
+ [InlineData("m +123", OperatorType.Plus, 0u, 2u, true)]
+ [InlineData("-123", OperatorType.Minus, 0u, 0u, true)]
+ [InlineData("*123", OperatorType.Multiply, 0u, 0u, true)]
+ [InlineData("/123", OperatorType.Divide, 0u, 0u, true)]
+ [InlineData("=123", OperatorType.Equal, 0u, 0u, true)]
+ [InlineData("<123", OperatorType.Less, 0u, 0u, true)]
+ [InlineData(">123", OperatorType.Greater, 0u, 0u, true)]
+ [InlineData("<=123", OperatorType.LessEqual, 0u, 0u, true)]
+ [InlineData(">=123", OperatorType.GreaterEqual, 0u, 0u, true)]
+ [InlineData("<>123", OperatorType.NotEqual, 0u, 0u, true)]
+ [InlineData(":=123", OperatorType.Assign, 0u, 0u, true)]
+ [InlineData("and 123", OperatorType.And, 0u, 0u, true)]
+ [InlineData("or123", OperatorType.Or, 0u, 0u, true)]
+ [InlineData("mod123", OperatorType.Mod, 0u, 0u, true)]
+
+ [InlineData("and123", OperatorType.And, 0u, 0u, false)]
+ [InlineData("andasd", OperatorType.And, 0u, 0u, false)]
+ [InlineData("andand", OperatorType.And, 0u, 0u, false)]
+ [InlineData("<><123", OperatorType.NotEqual, 0u, 0u, false)]
+ [InlineData("<><123", OperatorType.Less, 0u, 0u, false)]
+ [InlineData("<=<123", OperatorType.LessEqual, 0u, 0u, false)]
+ public void SmokeTest(string input, OperatorType type, uint expectedLinePos, uint expectedCharacterPos, bool expectedResult)
+ {
+ LinkedList content = Utils.GetLinkedList(input);
+ Assert.Equal(expectedResult, OperatorSemanticToken.TryParse(expectedLinePos, expectedCharacterPos, content.First!,
+ out OperatorSemanticToken? token));
+ if (expectedResult)
+ {
+ Assert.NotNull(token);
+ Assert.Equal(type, token.OperatorType);
+ Assert.Equal(expectedLinePos, token.LinePos);
+ Assert.Equal(expectedCharacterPos, token.CharacterPos);
+ }
+ else
+ {
+ Assert.Null(token);
+ }
+ }
+}
From 62b97aa3b6a9686b2d47b5cc840ca1537a0169a3 Mon Sep 17 00:00:00 2001
From: duqoo <92306417+duqoo@users.noreply.github.com>
Date: Wed, 13 Mar 2024 17:05:07 +0800
Subject: [PATCH 2/2] =?UTF-8?q?fix:=E6=B5=8B=E8=AF=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
Canon.Core/LexicalParser/SemanticToken.cs | 12 +++++++++++-
Canon.Tests/LexicalParserTests/NumberTests.cs | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/Canon.Core/LexicalParser/SemanticToken.cs b/Canon.Core/LexicalParser/SemanticToken.cs
index 358c291..72d8cf3 100644
--- a/Canon.Core/LexicalParser/SemanticToken.cs
+++ b/Canon.Core/LexicalParser/SemanticToken.cs
@@ -215,8 +215,9 @@ public class NumberSemanticToken : SemanticToken
bool hasDecimalPoint = false;
bool hasExponent = false;
+ bool hasMinusSign = false;
- while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E'))
+ while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E' || now.Value == '-' || now.Value == '+'))
{
if (now.Value == '.')
{
@@ -236,6 +237,15 @@ public class NumberSemanticToken : SemanticToken
hasExponent = true;
}
+ if (now.Value == '-' || now.Value == '+')
+ {
+ if (hasMinusSign)
+ {
+ break;
+ }
+ hasMinusSign = true;
+ }
+
buffer.Append(now.Value);
now = now.Next;
}
diff --git a/Canon.Tests/LexicalParserTests/NumberTests.cs b/Canon.Tests/LexicalParserTests/NumberTests.cs
index 9e8ab0f..eb2bd94 100644
--- a/Canon.Tests/LexicalParserTests/NumberTests.cs
+++ b/Canon.Tests/LexicalParserTests/NumberTests.cs
@@ -24,7 +24,7 @@ namespace Canon.Tests.LexicalParserTests
[InlineData("1E-7", 1E-7, NumberType.Real)]
[InlineData("1E", 0, NumberType.Real, false)]
[InlineData("abc", 0, NumberType.Integer, false)]
- [InlineData("123abc", 0, NumberType.Integer, false)]
+ [InlineData("123abc", 123, NumberType.Integer, true)]
public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true)
{
LinkedList content = Utils.GetLinkedList(input);