post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 词法分析器的基本功能(#12)

Browse Source 
Co-authored-by: jackfiled <xcrenchangjun@outlook.com>
Reviewed-on: PostGuard/Canon#12
Co-authored-by: Huaps <1183155719@qq.com>
Co-committed-by: Huaps <1183155719@qq.com>
This commit is contained in:
Huaps
2024-03-15 12:00:47 +08:00
committed by jackfiled
parent 99fdd6438b
commit fe35288cb5
9 changed files with 759 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Canon.Tests/LexicalParserTests/Array.cs
View File

@@ -1,6 +0,0 @@
namespace Canon.Tests.LexicalParserTests;
public class Array
{
}

12
Canon.Tests/LexicalParserTests/DelimiterTests.cs
View File

@@ -3,7 +3,6 @@ using Canon.Core.LexicalParser;
namespace Canon.Tests.LexicalParserTests;
public class DelimiterTests
{
[Theory]
@@ -17,11 +16,12 @@ public class DelimiterTests
[InlineData("]asd", DelimiterType.RightSquareBracket)]
public void SmokeTest(string input, DelimiterType type)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
Assert.True(DelimiterSemanticToken.TryParse(0, 0, content.First!,
out DelimiterSemanticToken? token));
Assert.NotNull(token);
Assert.Equal(type, token.DelimiterType);
SemanticToken token = tokens[0];
Assert.Equal(SemanticTokenType.Delimiter, token.TokenType);
DelimiterSemanticToken delimiterSemanticToken = (DelimiterSemanticToken)token;
Assert.Equal(type, delimiterSemanticToken.DelimiterType);
}
}

17
Canon.Tests/LexicalParserTests/IndentifierTests.cs
View File

@@ -18,20 +18,13 @@ namespace Canon.Tests.LexicalParserTests
[InlineData("identifier_with_special_chars@#", false)]
[InlineData("", false)]
[InlineData(" ", false)]
[InlineData("andand",false)]
[InlineData("andand", false)]
public void TestParseIdentifier(string input, bool expectedResult)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.Equal(expectedResult, IdentifierSemanticToken.TryParse(0, 0, content.First!,
out IdentifierSemanticToken? token));
if (expectedResult)
{
Assert.NotNull(token);
}
else
{
Assert.Null(token);
}
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
Assert.Equal(expectedResult, tokens.FirstOrDefault()?.TokenType == SemanticTokenType.Identifier);
}
}
}

11
Canon.Tests/LexicalParserTests/KeywordTypeTests.cs
View File

@@ -23,11 +23,12 @@ public class KeywordTypeTests
[InlineData("do", KeywordType.Do)]
public void SmokeTest(string input, KeywordType type)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
Assert.True(KeywordSemanticToken.TryParse(0, 0, content.First!,
out KeywordSemanticToken? token));
Assert.NotNull(token);
Assert.Equal(type, token.KeywordType);
SemanticToken token = tokens[0];
Assert.Equal(SemanticTokenType.Keyword, token.TokenType);
KeywordSemanticToken keywordSemanticToken = (KeywordSemanticToken)token;
Assert.Equal(type, keywordSemanticToken.KeywordType);
}
}

25
Canon.Tests/LexicalParserTests/NumberTests.cs
View File

@@ -25,21 +25,22 @@ namespace Canon.Tests.LexicalParserTests
[InlineData("1E", 0, NumberType.Real, false)]
[InlineData("abc", 0, NumberType.Integer, false)]
[InlineData("123abc", 123, NumberType.Integer, true)]
public void TestParseNumber(string input, double expected, NumberType expectedNumberType, bool expectedResult = true)
public void TestParseNumber(string input, double expected, NumberType expectedNumberType,
bool expectedResult = true)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.Equal(expectedResult, NumberSemanticToken.TryParse(0, 0, content.First!,
out NumberSemanticToken? token));
if (expectedResult)
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
if (!expectedResult)
{
Assert.NotNull(token);
Assert.Equal(expected, token.Value);
Assert.Equal(expectedNumberType, token.NumberType);
}
else
{
Assert.Null(token);
Assert.NotEqual(SemanticTokenType.Keyword, token.TokenType);
return;
}
Assert.Equal(SemanticTokenType.Number, token.TokenType);
NumberSemanticToken numberSemanticToken = (NumberSemanticToken)token;
Assert.Equal(expectedNumberType, numberSemanticToken.NumberType);
Assert.Equal(expected, numberSemanticToken.Value);
}
}
}

26
Canon.Tests/LexicalParserTests/OperatorTypeTests.cs
View File

@@ -7,9 +7,7 @@ public class OperatorTypeTests
{
[Theory]
[InlineData("+ 123", OperatorType.Plus)]
[InlineData("1 + 123", OperatorType.Plus)]
[InlineData("+123", OperatorType.Plus)]
[InlineData("m +123", OperatorType.Plus)]
[InlineData("-123", OperatorType.Minus)]
[InlineData("*123", OperatorType.Multiply)]
[InlineData("/123", OperatorType.Divide)]
@@ -22,20 +20,24 @@ public class OperatorTypeTests
[InlineData(":=123", OperatorType.Assign)]
public void ParseTest(string input, OperatorType result)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.True(OperatorSemanticToken.TryParse(0, 0,
content.First!, out OperatorSemanticToken? token));
Assert.Equal(result, token?.OperatorType);
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
Assert.Equal(SemanticTokenType.Operator, token.TokenType);
OperatorSemanticToken operatorSemanticToken = (OperatorSemanticToken)token;
Assert.Equal(result, operatorSemanticToken.OperatorType);
}
[Theory]
[InlineData("<><123")]
[InlineData("<=<123")]
[InlineData("1 + 123")]
[InlineData("m +123")]
public void ParseFailedTest(string input)
{
LinkedList<char> content = Utils.GetLinkedList(input);
Assert.False(OperatorSemanticToken.TryParse(0, 0,
content.First!, out OperatorSemanticToken? token));
Assert.Null(token);
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
Assert.NotEqual(SemanticTokenType.Operator, token.TokenType);
}
}