diff --git a/Canon.Core/LexicalParser/LexRules.cs b/Canon.Core/LexicalParser/LexRules.cs index d6d607b..434c2ed 100644 --- a/Canon.Core/LexicalParser/LexRules.cs +++ b/Canon.Core/LexicalParser/LexRules.cs @@ -1,4 +1,4 @@ -using Canon.Core.Enums; +using Canon.Core.Enums; namespace Canon.Core.LexicalParser; diff --git a/Canon.Core/LexicalParser/Lexer.cs b/Canon.Core/LexicalParser/Lexer.cs index a19dc5b..c6bf38a 100644 --- a/Canon.Core/LexicalParser/Lexer.cs +++ b/Canon.Core/LexicalParser/Lexer.cs @@ -10,7 +10,7 @@ public class Lexer : ILexer // 记录token private SemanticToken? _semanticToken; private readonly StringBuilder _tokenBuilder = new(); - private readonly List _tokens = []; + private List _tokens = []; // 状态机 private StateType _state = StateType.Start; @@ -25,6 +25,7 @@ public class Lexer : ILexer public IEnumerable Tokenize(ISourceReader reader) { _reader = reader; + _tokens = []; _state = StateType.Start; while (_state != StateType.Done) diff --git a/Canon.Tests/LexicalParserTests/CharacterTypeTests.cs b/Canon.Tests/LexicalParserTests/CharacterTypeTests.cs index 6f20401..d0a2179 100644 --- a/Canon.Tests/LexicalParserTests/CharacterTypeTests.cs +++ b/Canon.Tests/LexicalParserTests/CharacterTypeTests.cs @@ -38,8 +38,8 @@ namespace Canon.Tests.LexicalParserTests [Theory] //[InlineData("'\\x'", 1, 2, LexemeException.LexemeErrorType.InvalidEscapeSequence)] - [InlineData("'This is an unclosed string literal", 1, 36, LexemeErrorType.UnclosedStringLiteral)] - [InlineData("'This", 1, 6, LexemeErrorType.UnclosedStringLiteral)] + [InlineData("'This is an unclosed string literal", 1, 35, LexemeErrorType.UnclosedStringLiteral)] + [InlineData("'This", 1, 5, LexemeErrorType.UnclosedStringLiteral)] [InlineData("x @", 1, 3, LexemeErrorType.UnknownCharacterOrString)] //[InlineData("\"x\'", 1, 3, LexemeException.LexemeErrorType.UnclosedStringLiteral)] public void TestParseCharacterError(string input, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType) diff --git a/Canon.Tests/LexicalParserTests/ErrorSingleTests.cs b/Canon.Tests/LexicalParserTests/ErrorSingleTests.cs index bbc1a69..076b47b 100644 --- a/Canon.Tests/LexicalParserTests/ErrorSingleTests.cs +++ b/Canon.Tests/LexicalParserTests/ErrorSingleTests.cs @@ -18,7 +18,7 @@ namespace Canon.Tests.LexicalParserTests [Theory] [InlineData("program main; var a: integer; begin a := 3#; end.", 1, 43, LexemeErrorType.IllegalNumberFormat)] - [InlineData("char c = 'abc;", 1, 15, LexemeErrorType.UnclosedStringLiteral)] + [InlineData("char c = 'abc;", 1, 14, LexemeErrorType.UnclosedStringLiteral)] [InlineData("x := 10 @;", 1, 9, LexemeErrorType.UnknownCharacterOrString)] [InlineData("identifier_with_special_chars@#",1, 30, LexemeErrorType.UnknownCharacterOrString)] public void TestUnknownCharacterError(string pascalProgram, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType) diff --git a/Canon.Tests/LexicalParserTests/LexicalFileTests.cs b/Canon.Tests/LexicalParserTests/LexicalFileTests.cs index faa5a4a..8928030 100644 --- a/Canon.Tests/LexicalParserTests/LexicalFileTests.cs +++ b/Canon.Tests/LexicalParserTests/LexicalFileTests.cs @@ -45,8 +45,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper) SemanticTokenType.Delimiter, SemanticTokenType.Delimiter, SemanticTokenType.Keyword, - SemanticTokenType.Delimiter, - SemanticTokenType.End + SemanticTokenType.Delimiter ]); } @@ -140,42 +139,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper) SemanticTokenType.Delimiter, SemanticTokenType.Delimiter, SemanticTokenType.Keyword, - SemanticTokenType.Delimiter, - SemanticTokenType.End - ]); - } - - [Fact] - public void ReuseTest() - { - const string program1 = """ - program main; - begin - end. - """; - IEnumerable tokens = _lexer.Tokenize(new StringSourceReader(program1)); - - ValidateSemanticTokens(tokens, [ - SemanticTokenType.Keyword, - SemanticTokenType.Identifier, - SemanticTokenType.Delimiter, - SemanticTokenType.Keyword, - SemanticTokenType.Keyword, - SemanticTokenType.Delimiter - ]); - - const string test = "program begin end."; - - tokens = _lexer.Tokenize(new StringSourceReader(test)); - - ValidateSemanticTokens(tokens, [ - SemanticTokenType.Keyword, - SemanticTokenType.Keyword, - SemanticTokenType.Keyword, SemanticTokenType.Delimiter ]); } + [Fact] public void UnclosedCommentFirst() { @@ -193,7 +161,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper) testOutputHelper.WriteLine(ex.ToString()); Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType); Assert.Equal((uint)7, ex.Line); - Assert.Equal((uint)5, ex.CharPosition); + Assert.Equal((uint)4, ex.CharPosition); } [Fact] @@ -209,7 +177,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper) testOutputHelper.WriteLine(ex.ToString()); Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType); Assert.Equal((uint)4, ex.Line); - Assert.Equal((uint)26, ex.CharPosition); + Assert.Equal((uint)25, ex.CharPosition); } [Fact] @@ -313,7 +281,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper) private static void ValidateSemanticTokens(IEnumerable actualTokens, IEnumerable expectedTypes) { - foreach ((SemanticTokenType type, SemanticToken token) in expectedTypes.Zip(actualTokens)) + List types = [..expectedTypes, SemanticTokenType.End]; + List tokens = actualTokens.ToList(); + + Assert.Equal(types.Count, tokens.Count); + foreach ((SemanticTokenType type, SemanticToken token) in types.Zip(tokens)) { Assert.Equal(type, token.TokenType); }