fix: 词法位置报错 (#53)

Co-authored-by: Huaps <1183155719@qq.com>
Reviewed-on: PostGuard/Canon#53

Canon.Core/LexicalParser/LexRules.cs

@@ -1,4 +1,4 @@
-using Canon.Core.Enums;
+using Canon.Core.Enums;
 
 namespace Canon.Core.LexicalParser;
 

Canon.Core/LexicalParser/Lexer.cs

@@ -10,7 +10,7 @@ public class Lexer : ILexer
     // 记录token
     private SemanticToken? _semanticToken;
     private readonly StringBuilder _tokenBuilder = new();
-    private readonly List<SemanticToken> _tokens = [];
+    private List<SemanticToken> _tokens = [];
 
     // 状态机
     private StateType _state = StateType.Start;
@@ -25,6 +25,7 @@ public class Lexer : ILexer
     public IEnumerable<SemanticToken> Tokenize(ISourceReader reader)
     {
         _reader = reader;
+        _tokens = [];
         _state = StateType.Start;
 
         while (_state != StateType.Done)

Canon.Tests/LexicalParserTests/CharacterTypeTests.cs

@@ -38,8 +38,8 @@ namespace Canon.Tests.LexicalParserTests
 
         [Theory]
         //[InlineData("'\\x'", 1, 2, LexemeException.LexemeErrorType.InvalidEscapeSequence)]
-        [InlineData("'This is an unclosed string literal", 1, 36, LexemeErrorType.UnclosedStringLiteral)]
+        [InlineData("'This is an unclosed string literal", 1, 35, LexemeErrorType.UnclosedStringLiteral)]
-        [InlineData("'This", 1, 6, LexemeErrorType.UnclosedStringLiteral)]
+        [InlineData("'This", 1, 5, LexemeErrorType.UnclosedStringLiteral)]
         [InlineData("x @", 1, 3, LexemeErrorType.UnknownCharacterOrString)]
         //[InlineData("\"x\'", 1, 3, LexemeException.LexemeErrorType.UnclosedStringLiteral)]
         public void TestParseCharacterError(string input,  uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)

Canon.Tests/LexicalParserTests/ErrorSingleTests.cs

@@ -18,7 +18,7 @@ namespace Canon.Tests.LexicalParserTests
 
         [Theory]
         [InlineData("program main; var a: integer; begin a := 3#; end.", 1, 43, LexemeErrorType.IllegalNumberFormat)]
-        [InlineData("char c = 'abc;", 1, 15, LexemeErrorType.UnclosedStringLiteral)]
+        [InlineData("char c = 'abc;", 1, 14, LexemeErrorType.UnclosedStringLiteral)]
         [InlineData("x := 10 @;", 1, 9, LexemeErrorType.UnknownCharacterOrString)]
         [InlineData("identifier_with_special_chars@#",1, 30, LexemeErrorType.UnknownCharacterOrString)]
         public void TestUnknownCharacterError(string pascalProgram, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)

Canon.Tests/LexicalParserTests/LexicalFileTests.cs

@@ -45,8 +45,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
             SemanticTokenType.Delimiter,
             SemanticTokenType.Delimiter,
             SemanticTokenType.Keyword,
-            SemanticTokenType.Delimiter,
+            SemanticTokenType.Delimiter
-            SemanticTokenType.End
         ]);
     }
 
@@ -140,42 +139,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
             SemanticTokenType.Delimiter,
             SemanticTokenType.Delimiter,
             SemanticTokenType.Keyword,
-            SemanticTokenType.Delimiter,
-            SemanticTokenType.End
-        ]);
-    }
-
-    [Fact]
-    public void ReuseTest()
-    {
-        const string program1 = """
-                               program main;
-                               begin
-                               end.
-                               """;
-        IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(program1));
-
-        ValidateSemanticTokens(tokens, [
-            SemanticTokenType.Keyword,
-            SemanticTokenType.Identifier,
-            SemanticTokenType.Delimiter,
-            SemanticTokenType.Keyword,
-            SemanticTokenType.Keyword,
-            SemanticTokenType.Delimiter
-        ]);
-
-        const string test = "program begin end.";
-
-        tokens = _lexer.Tokenize(new StringSourceReader(test));
-
-        ValidateSemanticTokens(tokens, [
-            SemanticTokenType.Keyword,
-            SemanticTokenType.Keyword,
-            SemanticTokenType.Keyword,
             SemanticTokenType.Delimiter
         ]);
     }
 
+
     [Fact]
     public void UnclosedCommentFirst()
     {
@@ -193,7 +161,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
         testOutputHelper.WriteLine(ex.ToString());
         Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
         Assert.Equal((uint)7, ex.Line);
-        Assert.Equal((uint)5, ex.CharPosition);
+        Assert.Equal((uint)4, ex.CharPosition);
     }
 
     [Fact]
@@ -209,7 +177,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
         testOutputHelper.WriteLine(ex.ToString());
         Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
         Assert.Equal((uint)4, ex.Line);
-        Assert.Equal((uint)26, ex.CharPosition);
+        Assert.Equal((uint)25, ex.CharPosition);
     }
 
     [Fact]
@@ -313,7 +281,11 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
     private static void ValidateSemanticTokens(IEnumerable<SemanticToken> actualTokens,
         IEnumerable<SemanticTokenType> expectedTypes)
     {
-        foreach ((SemanticTokenType type, SemanticToken token) in expectedTypes.Zip(actualTokens))
+        List<SemanticTokenType> types = [..expectedTypes, SemanticTokenType.End];
+        List<SemanticToken> tokens = actualTokens.ToList();
+
+        Assert.Equal(types.Count, tokens.Count);
+        foreach ((SemanticTokenType type, SemanticToken token) in types.Zip(tokens))
         {
             Assert.Equal(type, token.TokenType);
         }