post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加了对部分类型的测试样例，修改添加了NumberSemanticToken

Browse Source
This commit is contained in:
duqoo 2024-03-13 16:19:07 +08:00
parent 35aec34a8e
commit 7ea97ea97e
7 changed files with 226 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Canon.Core/Enums/SemanticEnums.cs
View File

@ -68,3 +68,10 @@ public enum OperatorType
Or,
Assign
}
public enum NumberType
{
Integer,
Real,
Hex
}

48
Canon.Core/LexicalParser/SemanticToken.cs
View File

@ -4,6 +4,8 @@ namespace Canon.Core.LexicalParser;
using Enums;
using System.Text;
/// <summary>
/// 词法记号基类
/// </summary>
@ -198,13 +200,59 @@ public class OperatorSemanticToken : SemanticToken
/// <summary>
/// 数值类型记号
/// </summary>
/// TODO：进制表示（只有$1的十六进制表示）
public class NumberSemanticToken : SemanticToken
{
public override SemanticTokenType TokenType => SemanticTokenType.Number;
public required NumberType NumberType { get; init; }
public double Value { get; private init; }
public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
out NumberSemanticToken? token)
{
StringBuilder buffer = new();
bool hasDecimalPoint = false;
bool hasExponent = false;
while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E'))
{
if (now.Value == '.')
{
if (hasDecimalPoint)
{
break;
}
hasDecimalPoint = true;
}
if (now.Value == 'e' || now.Value == 'E')
{
if (hasExponent)
{
break;
}
hasExponent = true;
}
buffer.Append(now.Value);
now = now.Next;
}
if (double.TryParse(buffer.ToString(), out double value))
{
token = new NumberSemanticToken
{
LinePos = linePos,
CharacterPos = characterPos,
LiteralValue = buffer.ToString(),
Value = value,
NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer
};
return true;
}
token = null;
return false;
}

6
Canon.Tests/LexicalParserTests/Array.cs Normal file
View File

@ -0,0 +1,6 @@
namespace Canon.Tests.LexicalParserTests;
public class Array
{
}

37
Canon.Tests/LexicalParserTests/IndentifierTests.cs Normal file
View File

@ -0,0 +1,37 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
using Xunit;
namespace Canon.Tests.LexicalParserTests
{
public class IdentifierTests
{
[Theory]
[InlineData("identifier", true)]
[InlineData("_identifier", true)]
[InlineData("identifier123", true)]
[InlineData("123identifier", false)]
[InlineData("identifier_with_underscores", true)]
[InlineData("IdentifierWithCamelCase", true)]
[InlineData("identifier-with-hyphen", false)]
[InlineData("identifier with spaces", false)]
[InlineData("identifier_with_special_chars@#", false)]
[InlineData("", false)]
[InlineData(" ", false)]
[InlineData("andand",false)]
public void TestParseIdentifier(string input, bool expectedResult)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!,
out IdentifierSemanticToken? token));
if (expectedResult)
{
Assert.NotNull(token);
}
else
{
Assert.Null(token);
}
}
}
}

33
Canon.Tests/LexicalParserTests/KeywordTypeTests.cs Normal file
View File

@ -0,0 +1,33 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Tests.LexicalParserTests;
public class KeywordTypeTests
{
[Theory]
[InlineData("program", KeywordType.Program)]
[InlineData("const", KeywordType.Const)]
[InlineData("var", KeywordType.Var)]
[InlineData("procedure", KeywordType.Procedure)]
[InlineData("function", KeywordType.Function)]
[InlineData("begin", KeywordType.Begin)]
[InlineData("end", KeywordType.End)]
[InlineData("array", KeywordType.Array)]
[InlineData("of", KeywordType.Of)]
[InlineData("if", KeywordType.If)]
[InlineData("then", KeywordType.Then)]
[InlineData("else", KeywordType.Else)]
[InlineData("for", KeywordType.For)]
[InlineData("to", KeywordType.To)]
[InlineData("do", KeywordType.Do)]
public void SmokeTest(string input, KeywordType type)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!,
out KeywordSemanticToken? token));
Assert.NotNull(token);
Assert.Equal(type, token.KeywordType);
}
}

45
Canon.Tests/LexicalParserTests/NumberTests.cs Normal file
View File

@ -0,0 +1,45 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Tests.LexicalParserTests
{
public class NumberTests
{
[Theory]
[InlineData("123", 123, NumberType.Integer)]
[InlineData("0", 0, NumberType.Integer)]
[InlineData("-123", -123, NumberType.Integer)]
[InlineData("1.23", 1.23, NumberType.Real)]
[InlineData("-1.23", -1.23, NumberType.Real)]
[InlineData("0.0", 0.0, NumberType.Real)]
[InlineData("1e7", 1e7, NumberType.Real)]
[InlineData("1E7", 1E7, NumberType.Real)]
[InlineData("1.23e-7", 1.23e-7, NumberType.Real)]
[InlineData("1.23E-7", 1.23E-7, NumberType.Real)]
[InlineData("1234567890", 1234567890, NumberType.Integer)]
[InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)]
[InlineData("-1234567890", -1234567890, NumberType.Integer)]
[InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)]
[InlineData("1e-7", 1e-7, NumberType.Real)]
[InlineData("1E-7", 1E-7, NumberType.Real)]
[InlineData("1E", 0, NumberType.Real, false)]
[InlineData("abc", 0, NumberType.Integer, false)]
[InlineData("123abc", 0, NumberType.Integer, false)]
public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!,
out NumberSemanticToken? token));
if (expectedResult)
{
Assert.NotNull(token);
Assert.Equal(expected, token.Value);
Assert.Equal(expectedNumberType, token.NumberType);
}
else
{
Assert.Null(token);
}
}
}
}

50
Canon.Tests/LexicalParserTests/OperatorTypeTests.cs Normal file
View File

@ -0,0 +1,50 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Tests.LexicalParserTests;
public class OperatorTypeTests
{
[Theory]
[InlineData("+ 123", OperatorType.Plus, 0u, 0u, true)]
[InlineData("1 + 123", OperatorType.Plus, 0u, 2u, true)]
[InlineData("+123", OperatorType.Plus, 0u, 0u, true)]
[InlineData("m +123", OperatorType.Plus, 0u, 2u, true)]
[InlineData("-123", OperatorType.Minus, 0u, 0u, true)]
[InlineData("*123", OperatorType.Multiply, 0u, 0u, true)]
[InlineData("/123", OperatorType.Divide, 0u, 0u, true)]
[InlineData("=123", OperatorType.Equal, 0u, 0u, true)]
[InlineData("<123", OperatorType.Less, 0u, 0u, true)]
[InlineData(">123", OperatorType.Greater, 0u, 0u, true)]
[InlineData("<=123", OperatorType.LessEqual, 0u, 0u, true)]
[InlineData(">=123", OperatorType.GreaterEqual, 0u, 0u, true)]
[InlineData("<>123", OperatorType.NotEqual, 0u, 0u, true)]
[InlineData(":=123", OperatorType.Assign, 0u, 0u, true)]
[InlineData("and 123", OperatorType.And, 0u, 0u, true)]
[InlineData("or123", OperatorType.Or, 0u, 0u, true)]
[InlineData("mod123", OperatorType.Mod, 0u, 0u, true)]
[InlineData("and123", OperatorType.And, 0u, 0u, false)]
[InlineData("andasd", OperatorType.And, 0u, 0u, false)]
[InlineData("andand", OperatorType.And, 0u, 0u, false)]
[InlineData("<><123", OperatorType.NotEqual, 0u, 0u, false)]
[InlineData("<><123", OperatorType.Less, 0u, 0u, false)]
[InlineData("<=<123", OperatorType.LessEqual, 0u, 0u, false)]
public void SmokeTest(string input, OperatorType type, uint expectedLinePos, uint expectedCharacterPos, bool expectedResult)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.Equal(expectedResult, OperatorSemanticToken.TryParse(expectedLinePos, expectedCharacterPos, content.First!,
out OperatorSemanticToken? token));
if (expectedResult)
{
Assert.NotNull(token);
Assert.Equal(type, token.OperatorType);
Assert.Equal(expectedLinePos, token.LinePos);
Assert.Equal(expectedCharacterPos, token.CharacterPos);
}
else
{
Assert.Null(token);
}
}
}