misc: 删除冗余的代码 (#41)

Reviewed-on: PostGuard/Canon#41

.gitea/workflows/test.yaml

@@ -13,5 +13,7 @@ jobs:
                   path: ~/.nuget/packages
                   key: ${{ runner.os }}-nuget
                   save-always: true
+            - name: Build code
+              run: dotnet build
             - name: Run test code
               run: dotnet test

Canon.Core/LexicalParser/LexemeFactory.cs

@@ -4,60 +4,57 @@ namespace Canon.Core.LexicalParser;
 
 public static class LexemeFactory
 {
-
+    public static SemanticToken MakeToken(SemanticTokenType tokenType,string literal,uint line,uint chPos)
-    public static SemanticToken MakeToken(SemanticTokenType tokenType,string literal,uint _line,uint _chPos)
     {
         SemanticToken? token;
         switch (tokenType)
         {
             case SemanticTokenType.Character:
-                CharacterSemanticToken characterSemanticToken = new CharacterSemanticToken()
+                CharacterSemanticToken characterSemanticToken = new()
                 {
-                    LinePos = _line, CharacterPos = _chPos, LiteralValue = literal,
+                    LinePos = line, CharacterPos = chPos, LiteralValue = literal,
                 };
                 token = characterSemanticToken;
                 break;
             case SemanticTokenType.Identifier:
-                IdentifierSemanticToken identifierSemanticToken = new IdentifierSemanticToken()
+                IdentifierSemanticToken identifierSemanticToken = new()
                 {
-                    LinePos = _line, CharacterPos = _chPos, LiteralValue = literal,
+                    LinePos = line, CharacterPos = chPos, LiteralValue = literal,
                 };
                 token = identifierSemanticToken;
                 break;
             default:
-                throw new ArgumentOutOfRangeException(nameof(tokenType), tokenType, null);
+                throw new InvalidOperationException("Can only create Character or Identifier SemanticToken.");
         }
 
         return token;
-
-
     }
 
-    public static KeywordSemanticToken MakeToken(KeywordType keywordType,string literal,uint _line,uint _chPos)
+    public static KeywordSemanticToken MakeToken(KeywordType keywordType,string literal,uint line,uint chPos)
     {
-        KeywordSemanticToken keywordSemanticToken = new KeywordSemanticToken
+        KeywordSemanticToken keywordSemanticToken = new()
         {
-            LinePos = _line,
+            LinePos = line,
-            CharacterPos = _chPos,
+            CharacterPos = chPos,
             LiteralValue = literal,
             KeywordType = keywordType
         };
         return keywordSemanticToken;
     }
 
-    public static DelimiterSemanticToken MakeToken(DelimiterType delimiterType,string literal,uint _line,uint _chPos)
+    public static DelimiterSemanticToken MakeToken(DelimiterType delimiterType,string literal,uint line,uint chPos)
     {
-        DelimiterSemanticToken delimiterSemanticToken = new DelimiterSemanticToken()
+        DelimiterSemanticToken delimiterSemanticToken = new()
         {
-            LinePos = _line,
+            LinePos = line,
-            CharacterPos = _chPos,
+            CharacterPos = chPos,
             LiteralValue = literal,
             DelimiterType = delimiterType
         };
         return delimiterSemanticToken;
     }
 
-    public static NumberSemanticToken MakeToken(NumberType numberType,string literal,uint _line,uint _chPos)
+    public static NumberSemanticToken MakeToken(NumberType numberType,string literal,uint line,uint chPos)
     {
         string temp = literal;
         string result;
@@ -70,10 +67,10 @@ public static class LexemeFactory
             result = temp;
         }
 
-        NumberSemanticToken numberSemanticToken = new NumberSemanticToken()
+        NumberSemanticToken numberSemanticToken = new()
         {
-            LinePos = _line,
+            LinePos = line,
-            CharacterPos = _chPos,
+            CharacterPos = chPos,
             LiteralValue = result,
             NumberType = numberType
         };
@@ -81,12 +78,12 @@ public static class LexemeFactory
 
     }
 
-    public static OperatorSemanticToken MakeToken(OperatorType operatorType,string literal,uint _line,uint _chPos)
+    public static OperatorSemanticToken MakeToken(OperatorType operatorType,string literal,uint line,uint chPos)
     {
-        OperatorSemanticToken operatorSemanticToken = new OperatorSemanticToken()
+        OperatorSemanticToken operatorSemanticToken = new()
         {
-            LinePos = _line,
+            LinePos = line,
-            CharacterPos = _chPos,
+            CharacterPos = chPos,
             LiteralValue = literal,
             OperatorType = operatorType
         };

Canon.Core/LexicalParser/Lexer.cs

@@ -22,21 +22,10 @@ public class Lexer : ILexer
     private uint _line = 1;
     private uint _chPos;
 
-    // Token统计信息
-    private readonly Dictionary<SemanticTokenType, int> _tokenCount = new()
-    {
-        { SemanticTokenType.Keyword, 0 },
-        { SemanticTokenType.Number, 0 },
-        { SemanticTokenType.Operator, 0 },
-        { SemanticTokenType.Delimiter, 0 },
-        { SemanticTokenType.Identifier, 0 },
-        { SemanticTokenType.Character, 0 },
-        { SemanticTokenType.End, 0 }
-    };
-
     public IEnumerable<SemanticToken> Tokenize(ISourceReader reader)
     {
         _reader = reader;
+        _state = StateType.Start;
 
         while (_state != StateType.Done)
         {
@@ -526,7 +515,11 @@ public class Lexer : ILexer
                 break;
         }
 
-        AddToTokens(_semanticToken);
+        if (_semanticToken is null)
+        {
+            throw new InvalidOperationException();
+        }
+        _tokens.Add(_semanticToken);
         _state = StateType.Start;
     }
 
@@ -614,9 +607,6 @@ public class Lexer : ILexer
     private void AddToTokens(SemanticToken semanticToken)
     {
         _tokens.Add(semanticToken);
-        _tokenCount[semanticToken.TokenType]++;
-        Console.WriteLine($"<{semanticToken.TokenType}>");
-        Console.WriteLine(semanticToken.LiteralValue);
     }
 
     private void Cat()

Canon.Core/LexicalParser/SemanticToken.cs

@@ -54,12 +54,13 @@ public abstract class SemanticToken : IEquatable<SemanticToken>
     /// </summary>
     public static EndSemanticToken End => new()
     {
-        LinePos = 0, CharacterPos = 0, LiteralValue = string.Empty
+        LinePos = uint.MaxValue, CharacterPos = uint.MaxValue, LiteralValue = string.Empty
     };
 
     public override string ToString()
     {
-        return $"LinePos: {LinePos}, CharacterPos: {CharacterPos}, LiteralValue: {LiteralValue}, TokenType: {TokenType}";
+        return
+            $"LinePos: {LinePos}, CharacterPos: {CharacterPos}, LiteralValue: {LiteralValue}, TokenType: {TokenType}";
     }
 
     public bool Equals(SemanticToken? other)
@@ -93,13 +94,6 @@ public abstract class SemanticToken : IEquatable<SemanticToken>
 public class CharacterSemanticToken : SemanticToken
 {
     public override SemanticTokenType TokenType => SemanticTokenType.Character;
-
-    public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
-        out CharacterSemanticToken? token)
-    {
-        token = null;
-        return false;
-    }
 }
 
 /// <summary>
@@ -146,7 +140,7 @@ public class DelimiterSemanticToken : SemanticToken
 
     public override int GetHashCode()
     {
-        return base.GetHashCode() ^ this.DelimiterType.GetHashCode();
+        return base.GetHashCode() ^ DelimiterType.GetHashCode();
     }
 }
 
@@ -159,7 +153,8 @@ public class KeywordSemanticToken : SemanticToken
 
     public required KeywordType KeywordType { get; init; }
 
-    public static readonly Dictionary<string, KeywordType> KeywordTypes = new Dictionary<string, KeywordType>(StringComparer.OrdinalIgnoreCase)
+    public static readonly Dictionary<string, KeywordType> KeywordTypes =
+        new Dictionary<string, KeywordType>(StringComparer.OrdinalIgnoreCase)
         {
             { "program", KeywordType.Program },
             { "const", KeywordType.Const },
@@ -199,56 +194,6 @@ public class KeywordSemanticToken : SemanticToken
         }
     }
 
-    public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
-        out KeywordSemanticToken? token)
-    {
-        string buffer = new([now.Value]);
-
-        if (now.Next is null)
-        {
-            // 没有比两个字符更短的关键字
-            token = null;
-            return false;
-        }
-
-        now = now.Next;
-        buffer += now.Value;
-
-        switch (buffer)
-        {
-            case "do":
-                token = new KeywordSemanticToken
-                {
-                    LinePos = linePos,
-                    CharacterPos = characterPos,
-                    LiteralValue = "do",
-                    KeywordType = KeywordType.Do
-                };
-                return true;
-            case "Of":
-                token = new KeywordSemanticToken
-                {
-                    LinePos = linePos,
-                    CharacterPos = characterPos,
-                    LiteralValue = "of",
-                    KeywordType = KeywordType.Of
-                };
-                return true;
-            case "If":
-                token = new KeywordSemanticToken
-                {
-                    LinePos = linePos,
-                    CharacterPos = characterPos,
-                    LiteralValue = "if",
-                    KeywordType = KeywordType.If
-                };
-                return true;
-        }
-
-        token = null;
-        return false;
-    }
-
     public override int GetHashCode()
     {
         return base.GetHashCode() ^ this.KeywordType.GetHashCode();
@@ -291,16 +236,9 @@ public class OperatorSemanticToken : SemanticToken
         }
     }
 
-    public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
-        out OperatorSemanticToken? token)
-    {
-        token = null;
-        return false;
-    }
-
     public override int GetHashCode()
     {
-        return base.GetHashCode() ^ this.OperatorType.GetHashCode();
+        return base.GetHashCode() ^ OperatorType.GetHashCode();
     }
 }
 
@@ -315,7 +253,7 @@ public class NumberSemanticToken : SemanticToken
 
     public override int GetHashCode()
     {
-        return base.GetHashCode() ^ this.NumberType.GetHashCode();
+        return base.GetHashCode() ^ NumberType.GetHashCode();
     }
 }
 
@@ -326,17 +264,13 @@ public class IdentifierSemanticToken : SemanticToken
 {
     public override SemanticTokenType TokenType => SemanticTokenType.Identifier;
 
-    public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
+    /// <summary>
-        out IdentifierSemanticToken? token)
+    /// 标识符名称
-    {
+    /// </summary>
-        token = null;
+    public string IdentifierName => LiteralValue.ToLower();
-        return false;
-    }
 }
 
 public class EndSemanticToken : SemanticToken
 {
     public override SemanticTokenType TokenType => SemanticTokenType.End;
 }
-
-

Canon.Tests/LexicalParserTests/LexicalFileTests.cs

@@ -1,149 +1,16 @@
-using System.Text.RegularExpressions;
+using Canon.Core.Enums;
-using Canon.Core.Enums;
 using Canon.Core.Exceptions;
 using Canon.Core.LexicalParser;
-using Xunit.Abstractions;
 using Canon.Tests.Utils;
 using Canon.Core.Abstractions;
+using Xunit.Abstractions;
 
 namespace Canon.Tests.LexicalParserTests;
 
-public class LexicalFileTests
+public class LexicalFileTests(ITestOutputHelper testOutputHelper)
 {
-    private readonly ITestOutputHelper _testOutputHelper;
     private readonly ILexer _lexer = new Lexer();
 
-    public LexicalFileTests(ITestOutputHelper testOutputHelper)
-    {
-        _testOutputHelper = testOutputHelper;
-    }
-
-    //TODO: 基础的字符串匹配，因此变量名称不能被包含。手写一个存在包含情况的测试文件。
-    private static (int, int) FindNthPosition(string pascalProgram, string target, int occurrence)
-    {
-        int lineNumber = 0;
-        (int, int) nthPosition = (0, 0);
-        int foundCount = 0;
-        occurrence = occurrence + 1;
-
-        using (StringReader sr = new StringReader(pascalProgram))
-        {
-            string line;
-            while ((line = sr.ReadLine()) != null)
-            {
-                lineNumber++;
-                int columnNumber = -1;
-
-                // line = Regex.Replace(line, "'[^']*'", "$");
-
-                while ((columnNumber = line.IndexOf(target, columnNumber + 1, StringComparison.Ordinal)) != -1)
-                {
-                    foundCount++;
-                    if (foundCount == occurrence)
-                    {
-                        nthPosition = (lineNumber, columnNumber + target.Length);
-                        return nthPosition;
-                    }
-                }
-            }
-        }
-
-        if (nthPosition == (0, 0))
-        {
-            throw new Exception($"'{target}' not found in program.");
-        }
-
-        return nthPosition;
-    }
-
-    private void TestLexicalAnalysis(string pascalProgram, List<(string, SemanticTokenType, int)> stringLiterals)
-    {
-        var expectedTokens = new List<SemanticToken>();
-
-        foreach (var (literal, tokenType, skipCount) in stringLiterals)
-        {
-            var (line, column) = FindNthPosition(pascalProgram, literal, skipCount);
-            switch (tokenType)
-            {
-                case SemanticTokenType.Keyword:
-                    expectedTokens.Add(new KeywordSemanticToken
-                    {
-                        LinePos = (uint)line,
-                        CharacterPos = (uint)column,
-                        LiteralValue = literal,
-                        KeywordType = KeywordSemanticToken.GetKeywordTypeByKeyword(literal)
-                    });
-                    break;
-                case SemanticTokenType.Identifier:
-                    expectedTokens.Add(new IdentifierSemanticToken
-                    {
-                        LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal
-                    });
-                    break;
-                case SemanticTokenType.Delimiter:
-                    if (DelimiterSemanticToken.TryParse((uint)line, (uint)column, new LinkedListNode<char>(literal[0]),
-                            out var delimiterToken))
-                    {
-                        if (delimiterToken != null)
-                        {
-                            expectedTokens.Add(delimiterToken);
-                        }
-                    }
-
-                    break;
-                case SemanticTokenType.Operator:
-                    expectedTokens.Add(new OperatorSemanticToken
-                    {
-                        LinePos = (uint)line,
-                        CharacterPos = (uint)column,
-                        LiteralValue = literal,
-                        OperatorType = OperatorSemanticToken.GetOperatorTypeByOperator(literal)
-                    });
-                    break;
-                case SemanticTokenType.Character:
-                    expectedTokens.Add(new CharacterSemanticToken
-                    {
-                        LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal
-                    });
-                    break;
-                case SemanticTokenType.Number:
-                    expectedTokens.Add(new NumberSemanticToken
-                    {
-                        LinePos = (uint)line,
-                        CharacterPos = (uint)column,
-                        LiteralValue = literal,
-                        NumberType = NumberType.Integer
-                    });
-                    break;
-            }
-        }
-
-        expectedTokens = expectedTokens.OrderBy(token => token.LinePos).ThenBy(token => token.CharacterPos).ToList();
-        expectedTokens = expectedTokens.Select(token =>
-            token is CharacterSemanticToken characterToken && characterToken.LiteralValue == "hello, world!"
-                ? new CharacterSemanticToken
-                {
-                    LinePos = characterToken.LinePos,
-                    CharacterPos = characterToken.CharacterPos + 1,
-                    LiteralValue = characterToken.LiteralValue
-                }
-                : token).ToList();
-
-        IEnumerable<SemanticToken> tokensEnumerable = _lexer.Tokenize(new StringSourceReader(pascalProgram));
-        List<SemanticToken> tokens = tokensEnumerable.ToList();
-
-        var actualTokens = tokens;
-        for (int i = 0; i < expectedTokens.Count; i++)
-        {
-            _testOutputHelper.WriteLine($"Expect: {expectedTokens[i]}");
-            _testOutputHelper.WriteLine($"Actual: {actualTokens[i]}");
-            _testOutputHelper.WriteLine("----");
-            // Assert.Equal(expectedTokens[i], actualTokens[i]);
-        }
-
-        Assert.Equal(expectedTokens, actualTokens);
-    }
-
     [Fact]
     public void TestLexicalAnalysisFirst()
     {
@@ -157,30 +24,30 @@ public class LexicalFileTests
                                end.
                                """;
 
-        var stringLiterals = new List<(string, SemanticTokenType, int)>
+        IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram));
-        {
+        ValidateSemanticTokens(tokens, [
-            ("program", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Keyword,
-            ("HelloWorld", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Identifier,
-            (";", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Delimiter,
-            ("var", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Keyword,
-            ("message", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Identifier,
-            (":", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Delimiter,
-            ("string", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Identifier,
-            (";", SemanticTokenType.Delimiter, 1),
+            SemanticTokenType.Delimiter,
-            ("begin", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Keyword,
-            ("message", SemanticTokenType.Identifier, 1),
+            SemanticTokenType.Identifier,
-            (":=", SemanticTokenType.Operator, 0),
+            SemanticTokenType.Operator,
-            ("hello, world!", SemanticTokenType.Character, 0),
+            SemanticTokenType.Character,
-            (";", SemanticTokenType.Delimiter, 2),
+            SemanticTokenType.Delimiter,
-            ("writeln", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Identifier,
-            ("(", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Delimiter,
-            ("message", SemanticTokenType.Identifier, 2),
+            SemanticTokenType.Identifier,
-            (")", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Delimiter,
-            (";", SemanticTokenType.Delimiter, 3),
+            SemanticTokenType.Delimiter,
-            ("end", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Keyword,
-            (".", SemanticTokenType.Delimiter, 0)
+            SemanticTokenType.Delimiter,
-        };
+            SemanticTokenType.End
-        TestLexicalAnalysis(pascalProgram, stringLiterals);
+        ]);
     }
 
     [Fact]
@@ -196,30 +63,30 @@ public class LexicalFileTests
                                end.
                                """;
 
-        var stringLiterals = new List<(string, SemanticTokenType, int)>
+        IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram));
-        {
+
-            ("program", SemanticTokenType.Keyword, 0),
+        ValidateSemanticTokens(tokens, [
-            ("main", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Keyword,
-            (";", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Identifier,
-            ("var", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Delimiter,
-            ("ab", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Keyword,
-            (":", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Identifier,
-            ("integer", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Delimiter,
-            (";", SemanticTokenType.Delimiter, 1),
+            SemanticTokenType.Keyword,
-            ("begin", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Delimiter,
-            ("ab", SemanticTokenType.Identifier, 1),
+            SemanticTokenType.Keyword,
-            (":=", SemanticTokenType.Operator, 0),
+            SemanticTokenType.Identifier,
-            ("3", SemanticTokenType.Number, 0),
+            SemanticTokenType.Operator,
-            (";", SemanticTokenType.Delimiter, 2),
+            SemanticTokenType.Number,
-            ("write", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Delimiter,
-            ("(", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Identifier,
-            ("ab", SemanticTokenType.Identifier, 2),
+            SemanticTokenType.Delimiter,
-            (")", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Identifier,
-            (";", SemanticTokenType.Delimiter, 3),
+            SemanticTokenType.Delimiter,
-            ("end", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Delimiter,
-            (".", SemanticTokenType.Delimiter, 0)
+            SemanticTokenType.Keyword,
-        };
+            SemanticTokenType.Delimiter
-        TestLexicalAnalysis(pascalProgram, stringLiterals);
+        ]);
     }
 
     //带注释的测试
@@ -239,42 +106,74 @@ public class LexicalFileTests
                                end.
                                """;
 
-        var stringLiterals = new List<(string, SemanticTokenType, int)>
+        IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram));
+
+        ValidateSemanticTokens(tokens, [
+            SemanticTokenType.Keyword,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Keyword,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Keyword,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Keyword,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Operator,
+            SemanticTokenType.Number,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Operator,
+            SemanticTokenType.Number,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Operator,
+            SemanticTokenType.Number,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Operator,
+            SemanticTokenType.Identifier,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.Keyword,
+            SemanticTokenType.Delimiter,
+            SemanticTokenType.End
+        ]);
+    }
+
+    [Fact]
+    public void ReuseTest()
     {
-            ("program", SemanticTokenType.Keyword, 0),
+        const string program1 = """
-            ("main", SemanticTokenType.Identifier, 0),
+                               program main;
-            (";", SemanticTokenType.Delimiter, 0),
+                               begin
-            ("var", SemanticTokenType.Keyword, 0),
+                               end.
-            ("ab", SemanticTokenType.Identifier, 0),
+                               """;
-            (",", SemanticTokenType.Delimiter, 0),
+        IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(program1));
-            ("ba", SemanticTokenType.Identifier, 0),
+
-            (":", SemanticTokenType.Delimiter, 0),
+        ValidateSemanticTokens(tokens, [
-            ("integer", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Keyword,
-            (";", SemanticTokenType.Delimiter, 1),
+            SemanticTokenType.Identifier,
-            ("begin", SemanticTokenType.Keyword, 0),
+            SemanticTokenType.Delimiter,
-            ("ab", SemanticTokenType.Identifier, 1),
+            SemanticTokenType.Keyword,
-            (":=", SemanticTokenType.Operator, 0),
+            SemanticTokenType.Keyword,
-            ("3", SemanticTokenType.Number, 0),
+            SemanticTokenType.Delimiter
-            (";", SemanticTokenType.Delimiter, 2),
+        ]);
-            ("ba", SemanticTokenType.Identifier, 1),
+
-            (":=", SemanticTokenType.Operator, 1),
+        const string test = "program begin end.";
-            ("5", SemanticTokenType.Number, 0),
+
-            (";", SemanticTokenType.Delimiter, 3),
+        tokens = _lexer.Tokenize(new StringSourceReader(test));
-            ("ab", SemanticTokenType.Identifier, 2),
+
-            (":=", SemanticTokenType.Operator, 2),
+        ValidateSemanticTokens(tokens, [
-            ("5", SemanticTokenType.Number, 1),
+            SemanticTokenType.Keyword,
-            (";", SemanticTokenType.Delimiter, 4),
+            SemanticTokenType.Keyword,
-            ("write", SemanticTokenType.Identifier, 0),
+            SemanticTokenType.Keyword,
-            ("(", SemanticTokenType.Delimiter, 0),
+            SemanticTokenType.Delimiter
-            ("ab", SemanticTokenType.Identifier, 3),
+        ]);
-            ("+", SemanticTokenType.Operator, 0),
-            ("ba", SemanticTokenType.Identifier, 2),
-            (")", SemanticTokenType.Delimiter, 0),
-            (";", SemanticTokenType.Delimiter, 5),
-            ("end", SemanticTokenType.Keyword, 0),
-            (".", SemanticTokenType.Delimiter, 0)
-        };
-        TestLexicalAnalysis(pascalProgram, stringLiterals);
     }
 
     [Fact]
@@ -291,7 +190,7 @@ public class LexicalFileTests
                                """;
         var ex = Assert.Throws<LexemeException>(() => _lexer.Tokenize(new StringSourceReader(pascalProgram)).ToList());
         //打印exception信息
-        _testOutputHelper.WriteLine(ex.ToString());
+        testOutputHelper.WriteLine(ex.ToString());
         Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
         Assert.Equal((uint)7, ex.Line);
         Assert.Equal((uint)5, ex.CharPosition);
@@ -307,7 +206,7 @@ public class LexicalFileTests
                                program CommentNotClosed;
                                """;
         var ex = Assert.Throws<LexemeException>(() => _lexer.Tokenize(new StringSourceReader(pascalProgram)).ToList());
-        _testOutputHelper.WriteLine(ex.ToString());
+        testOutputHelper.WriteLine(ex.ToString());
         Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
         Assert.Equal((uint)4, ex.Line);
         Assert.Equal((uint)26, ex.CharPosition);
@@ -410,4 +309,13 @@ public class LexicalFileTests
         List<SemanticToken> tokens = tokensEnumerable.ToList();
         Assert.NotNull(tokens);
     }
+
+    private static void ValidateSemanticTokens(IEnumerable<SemanticToken> actualTokens,
+        IEnumerable<SemanticTokenType> expectedTypes)
+    {
+        foreach ((SemanticTokenType type, SemanticToken token) in expectedTypes.Zip(actualTokens))
+        {
+            Assert.Equal(type, token.TokenType);
+        }
+    }
 }