post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 词法位置报错 (#53)

Browse Source 
Co-authored-by: Huaps <1183155719@qq.com>
Reviewed-on: PostGuard/Canon#53
This commit is contained in:
jackfiled 2024-04-24 11:01:45 +08:00
parent d381f56e1d
commit 447a791793
5 changed files with 15 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Canon.Core/LexicalParser/LexRules.cs
View File

@ -1,4 +1,4 @@
using Canon.Core.Enums;
using Canon.Core.Enums;
namespace Canon.Core.LexicalParser;

3
Canon.Core/LexicalParser/Lexer.cs
View File

@ -10,7 +10,7 @@ public class Lexer : ILexer
// 记录token
private SemanticToken? _semanticToken;
private readonly StringBuilder _tokenBuilder = new();
private readonly List<SemanticToken> _tokens = [];
private List<SemanticToken> _tokens = [];
// 状态机
private StateType _state = StateType.Start;
@ -25,6 +25,7 @@ public class Lexer : ILexer
public IEnumerable<SemanticToken> Tokenize(ISourceReader reader)
{
_reader = reader;
_tokens = [];
_state = StateType.Start;
while (_state != StateType.Done)

4
Canon.Tests/LexicalParserTests/CharacterTypeTests.cs
View File

@ -38,8 +38,8 @@ namespace Canon.Tests.LexicalParserTests
[Theory]
//[InlineData("'\\x'", 1, 2, LexemeException.LexemeErrorType.InvalidEscapeSequence)]
[InlineData("'This is an unclosed string literal", 1, 36, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("'This", 1, 6, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("'This is an unclosed string literal", 1, 35, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("'This", 1, 5, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("x @", 1, 3, LexemeErrorType.UnknownCharacterOrString)]
//[InlineData("\"x\'", 1, 3, LexemeException.LexemeErrorType.UnclosedStringLiteral)]
public void TestParseCharacterError(string input, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)

2
Canon.Tests/LexicalParserTests/ErrorSingleTests.cs
View File

@ -18,7 +18,7 @@ namespace Canon.Tests.LexicalParserTests
[Theory]
[InlineData("program main; var a: integer; begin a := 3#; end.", 1, 43, LexemeErrorType.IllegalNumberFormat)]
[InlineData("char c = 'abc;", 1, 15, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("char c = 'abc;", 1, 14, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("x := 10 @;", 1, 9, LexemeErrorType.UnknownCharacterOrString)]
[InlineData("identifier_with_special_chars@#",1, 30, LexemeErrorType.UnknownCharacterOrString)]
public void TestUnknownCharacterError(string pascalProgram, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)

46
Canon.Tests/LexicalParserTests/LexicalFileTests.cs
View File

@ -45,8 +45,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
SemanticTokenType.Delimiter,
SemanticTokenType.Delimiter,
SemanticTokenType.Keyword,
SemanticTokenType.Delimiter,
SemanticTokenType.End
SemanticTokenType.Delimiter
]);
}
@ -140,42 +139,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
SemanticTokenType.Delimiter,
SemanticTokenType.Delimiter,
SemanticTokenType.Keyword,
SemanticTokenType.Delimiter,
SemanticTokenType.End
]);
}
[Fact]
public void ReuseTest()
{
const string program1 = """
program main;
begin
end.
""";
IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(program1));
ValidateSemanticTokens(tokens, [
SemanticTokenType.Keyword,
SemanticTokenType.Identifier,
SemanticTokenType.Delimiter,
SemanticTokenType.Keyword,
SemanticTokenType.Keyword,
SemanticTokenType.Delimiter
]);
const string test = "program begin end.";
tokens = _lexer.Tokenize(new StringSourceReader(test));
ValidateSemanticTokens(tokens, [
SemanticTokenType.Keyword,
SemanticTokenType.Keyword,
SemanticTokenType.Keyword,
SemanticTokenType.Delimiter
]);
}
[Fact]
public void UnclosedCommentFirst()
{
@ -193,7 +161,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
Assert.Equal((uint)7, ex.Line);
Assert.Equal((uint)5, ex.CharPosition);
Assert.Equal((uint)4, ex.CharPosition);
}
[Fact]
@ -209,7 +177,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
Assert.Equal((uint)4, ex.Line);
Assert.Equal((uint)26, ex.CharPosition);
Assert.Equal((uint)25, ex.CharPosition);
}
[Fact]
@ -313,7 +281,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
private static void ValidateSemanticTokens(IEnumerable<SemanticToken> actualTokens,
IEnumerable<SemanticTokenType> expectedTypes)
{
foreach ((SemanticTokenType type, SemanticToken token) in expectedTypes.Zip(actualTokens))
List<SemanticTokenType> types = [..expectedTypes, SemanticTokenType.End];
List<SemanticToken> tokens = actualTokens.ToList();
Assert.Equal(types.Count, tokens.Count);
foreach ((SemanticTokenType type, SemanticToken token) in types.Zip(tokens))
{
Assert.Equal(type, token.TokenType);
}