From 0fdfef8854af25fe938359b348f191171290d540 Mon Sep 17 00:00:00 2001 From: jackfiled Date: Sat, 20 Apr 2024 11:48:05 +0800 Subject: [PATCH] =?UTF-8?q?misc:=20=E5=88=A0=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=20(#41)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-on: https://git.rrricardo.top/PostGuard/Canon/pulls/41 --- .gitea/workflows/test.yaml | 2 + Canon.Core/LexicalParser/LexemeFactory.cs | 47 ++- Canon.Core/LexicalParser/Lexer.cs | 22 +- Canon.Core/LexicalParser/SemanticToken.cs | 142 ++----- .../LexicalParserTests/LexicalFileTests.cs | 352 +++++++----------- 5 files changed, 198 insertions(+), 367 deletions(-) diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml index d5accf2..401dfa0 100644 --- a/.gitea/workflows/test.yaml +++ b/.gitea/workflows/test.yaml @@ -13,5 +13,7 @@ jobs: path: ~/.nuget/packages key: ${{ runner.os }}-nuget save-always: true + - name: Build code + run: dotnet build - name: Run test code run: dotnet test diff --git a/Canon.Core/LexicalParser/LexemeFactory.cs b/Canon.Core/LexicalParser/LexemeFactory.cs index c612dcd..2b7957f 100644 --- a/Canon.Core/LexicalParser/LexemeFactory.cs +++ b/Canon.Core/LexicalParser/LexemeFactory.cs @@ -4,60 +4,57 @@ namespace Canon.Core.LexicalParser; public static class LexemeFactory { - - public static SemanticToken MakeToken(SemanticTokenType tokenType,string literal,uint _line,uint _chPos) + public static SemanticToken MakeToken(SemanticTokenType tokenType,string literal,uint line,uint chPos) { SemanticToken? token; switch (tokenType) { case SemanticTokenType.Character: - CharacterSemanticToken characterSemanticToken = new CharacterSemanticToken() + CharacterSemanticToken characterSemanticToken = new() { - LinePos = _line, CharacterPos = _chPos, LiteralValue = literal, + LinePos = line, CharacterPos = chPos, LiteralValue = literal, }; token = characterSemanticToken; break; case SemanticTokenType.Identifier: - IdentifierSemanticToken identifierSemanticToken = new IdentifierSemanticToken() + IdentifierSemanticToken identifierSemanticToken = new() { - LinePos = _line, CharacterPos = _chPos, LiteralValue = literal, + LinePos = line, CharacterPos = chPos, LiteralValue = literal, }; token = identifierSemanticToken; break; default: - throw new ArgumentOutOfRangeException(nameof(tokenType), tokenType, null); + throw new InvalidOperationException("Can only create Character or Identifier SemanticToken."); } return token; - - } - public static KeywordSemanticToken MakeToken(KeywordType keywordType,string literal,uint _line,uint _chPos) + public static KeywordSemanticToken MakeToken(KeywordType keywordType,string literal,uint line,uint chPos) { - KeywordSemanticToken keywordSemanticToken = new KeywordSemanticToken + KeywordSemanticToken keywordSemanticToken = new() { - LinePos = _line, - CharacterPos = _chPos, + LinePos = line, + CharacterPos = chPos, LiteralValue = literal, KeywordType = keywordType }; return keywordSemanticToken; } - public static DelimiterSemanticToken MakeToken(DelimiterType delimiterType,string literal,uint _line,uint _chPos) + public static DelimiterSemanticToken MakeToken(DelimiterType delimiterType,string literal,uint line,uint chPos) { - DelimiterSemanticToken delimiterSemanticToken = new DelimiterSemanticToken() + DelimiterSemanticToken delimiterSemanticToken = new() { - LinePos = _line, - CharacterPos = _chPos, + LinePos = line, + CharacterPos = chPos, LiteralValue = literal, DelimiterType = delimiterType }; return delimiterSemanticToken; } - public static NumberSemanticToken MakeToken(NumberType numberType,string literal,uint _line,uint _chPos) + public static NumberSemanticToken MakeToken(NumberType numberType,string literal,uint line,uint chPos) { string temp = literal; string result; @@ -70,10 +67,10 @@ public static class LexemeFactory result = temp; } - NumberSemanticToken numberSemanticToken = new NumberSemanticToken() + NumberSemanticToken numberSemanticToken = new() { - LinePos = _line, - CharacterPos = _chPos, + LinePos = line, + CharacterPos = chPos, LiteralValue = result, NumberType = numberType }; @@ -81,12 +78,12 @@ public static class LexemeFactory } - public static OperatorSemanticToken MakeToken(OperatorType operatorType,string literal,uint _line,uint _chPos) + public static OperatorSemanticToken MakeToken(OperatorType operatorType,string literal,uint line,uint chPos) { - OperatorSemanticToken operatorSemanticToken = new OperatorSemanticToken() + OperatorSemanticToken operatorSemanticToken = new() { - LinePos = _line, - CharacterPos = _chPos, + LinePos = line, + CharacterPos = chPos, LiteralValue = literal, OperatorType = operatorType }; diff --git a/Canon.Core/LexicalParser/Lexer.cs b/Canon.Core/LexicalParser/Lexer.cs index edcd595..068c297 100644 --- a/Canon.Core/LexicalParser/Lexer.cs +++ b/Canon.Core/LexicalParser/Lexer.cs @@ -22,21 +22,10 @@ public class Lexer : ILexer private uint _line = 1; private uint _chPos; - // Token统计信息 - private readonly Dictionary _tokenCount = new() - { - { SemanticTokenType.Keyword, 0 }, - { SemanticTokenType.Number, 0 }, - { SemanticTokenType.Operator, 0 }, - { SemanticTokenType.Delimiter, 0 }, - { SemanticTokenType.Identifier, 0 }, - { SemanticTokenType.Character, 0 }, - { SemanticTokenType.End, 0 } - }; - public IEnumerable Tokenize(ISourceReader reader) { _reader = reader; + _state = StateType.Start; while (_state != StateType.Done) { @@ -526,7 +515,11 @@ public class Lexer : ILexer break; } - AddToTokens(_semanticToken); + if (_semanticToken is null) + { + throw new InvalidOperationException(); + } + _tokens.Add(_semanticToken); _state = StateType.Start; } @@ -614,9 +607,6 @@ public class Lexer : ILexer private void AddToTokens(SemanticToken semanticToken) { _tokens.Add(semanticToken); - _tokenCount[semanticToken.TokenType]++; - Console.WriteLine($"<{semanticToken.TokenType}>"); - Console.WriteLine(semanticToken.LiteralValue); } private void Cat() diff --git a/Canon.Core/LexicalParser/SemanticToken.cs b/Canon.Core/LexicalParser/SemanticToken.cs index bc4b0c9..7866b35 100644 --- a/Canon.Core/LexicalParser/SemanticToken.cs +++ b/Canon.Core/LexicalParser/SemanticToken.cs @@ -54,12 +54,13 @@ public abstract class SemanticToken : IEquatable /// public static EndSemanticToken End => new() { - LinePos = 0, CharacterPos = 0, LiteralValue = string.Empty + LinePos = uint.MaxValue, CharacterPos = uint.MaxValue, LiteralValue = string.Empty }; public override string ToString() { - return $"LinePos: {LinePos}, CharacterPos: {CharacterPos}, LiteralValue: {LiteralValue}, TokenType: {TokenType}"; + return + $"LinePos: {LinePos}, CharacterPos: {CharacterPos}, LiteralValue: {LiteralValue}, TokenType: {TokenType}"; } public bool Equals(SemanticToken? other) @@ -93,13 +94,6 @@ public abstract class SemanticToken : IEquatable public class CharacterSemanticToken : SemanticToken { public override SemanticTokenType TokenType => SemanticTokenType.Character; - - public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, - out CharacterSemanticToken? token) - { - token = null; - return false; - } } /// @@ -146,7 +140,7 @@ public class DelimiterSemanticToken : SemanticToken public override int GetHashCode() { - return base.GetHashCode() ^ this.DelimiterType.GetHashCode(); + return base.GetHashCode() ^ DelimiterType.GetHashCode(); } } @@ -159,33 +153,34 @@ public class KeywordSemanticToken : SemanticToken public required KeywordType KeywordType { get; init; } - public static readonly Dictionary KeywordTypes = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - { "program", KeywordType.Program }, - { "const", KeywordType.Const }, - { "var", KeywordType.Var }, - { "procedure", KeywordType.Procedure }, - { "function", KeywordType.Function }, - { "begin", KeywordType.Begin }, - { "end", KeywordType.End }, - { "array", KeywordType.Array }, - { "of", KeywordType.Of }, - { "if", KeywordType.If }, - { "then", KeywordType.Then }, - { "else", KeywordType.Else }, - { "for", KeywordType.For }, - { "to", KeywordType.To }, - { "do", KeywordType.Do }, - { "integer", KeywordType.Integer }, - { "real", KeywordType.Real }, - { "boolean", KeywordType.Boolean }, - { "character", KeywordType.Character }, - { "div", KeywordType.Divide }, // 注意: Pascal 使用 'div' 而不是 '/' - { "not", KeywordType.Not }, - { "mod", KeywordType.Mod }, - { "and", KeywordType.And }, - { "or", KeywordType.Or } - }; + public static readonly Dictionary KeywordTypes = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "program", KeywordType.Program }, + { "const", KeywordType.Const }, + { "var", KeywordType.Var }, + { "procedure", KeywordType.Procedure }, + { "function", KeywordType.Function }, + { "begin", KeywordType.Begin }, + { "end", KeywordType.End }, + { "array", KeywordType.Array }, + { "of", KeywordType.Of }, + { "if", KeywordType.If }, + { "then", KeywordType.Then }, + { "else", KeywordType.Else }, + { "for", KeywordType.For }, + { "to", KeywordType.To }, + { "do", KeywordType.Do }, + { "integer", KeywordType.Integer }, + { "real", KeywordType.Real }, + { "boolean", KeywordType.Boolean }, + { "character", KeywordType.Character }, + { "div", KeywordType.Divide }, // 注意: Pascal 使用 'div' 而不是 '/' + { "not", KeywordType.Not }, + { "mod", KeywordType.Mod }, + { "and", KeywordType.And }, + { "or", KeywordType.Or } + }; public static KeywordType GetKeywordTypeByKeyword(string keyword) { @@ -199,56 +194,6 @@ public class KeywordSemanticToken : SemanticToken } } - public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, - out KeywordSemanticToken? token) - { - string buffer = new([now.Value]); - - if (now.Next is null) - { - // 没有比两个字符更短的关键字 - token = null; - return false; - } - - now = now.Next; - buffer += now.Value; - - switch (buffer) - { - case "do": - token = new KeywordSemanticToken - { - LinePos = linePos, - CharacterPos = characterPos, - LiteralValue = "do", - KeywordType = KeywordType.Do - }; - return true; - case "Of": - token = new KeywordSemanticToken - { - LinePos = linePos, - CharacterPos = characterPos, - LiteralValue = "of", - KeywordType = KeywordType.Of - }; - return true; - case "If": - token = new KeywordSemanticToken - { - LinePos = linePos, - CharacterPos = characterPos, - LiteralValue = "if", - KeywordType = KeywordType.If - }; - return true; - } - - token = null; - return false; - } - public override int GetHashCode() { return base.GetHashCode() ^ this.KeywordType.GetHashCode(); @@ -291,16 +236,9 @@ public class OperatorSemanticToken : SemanticToken } } - public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, - out OperatorSemanticToken? token) - { - token = null; - return false; - } - public override int GetHashCode() { - return base.GetHashCode() ^ this.OperatorType.GetHashCode(); + return base.GetHashCode() ^ OperatorType.GetHashCode(); } } @@ -315,7 +253,7 @@ public class NumberSemanticToken : SemanticToken public override int GetHashCode() { - return base.GetHashCode() ^ this.NumberType.GetHashCode(); + return base.GetHashCode() ^ NumberType.GetHashCode(); } } @@ -326,17 +264,13 @@ public class IdentifierSemanticToken : SemanticToken { public override SemanticTokenType TokenType => SemanticTokenType.Identifier; - public static bool TryParse(uint linePos, uint characterPos, LinkedListNode now, - out IdentifierSemanticToken? token) - { - token = null; - return false; - } + /// + /// 标识符名称 + /// + public string IdentifierName => LiteralValue.ToLower(); } public class EndSemanticToken : SemanticToken { public override SemanticTokenType TokenType => SemanticTokenType.End; } - - diff --git a/Canon.Tests/LexicalParserTests/LexicalFileTests.cs b/Canon.Tests/LexicalParserTests/LexicalFileTests.cs index f9b2978..faa5a4a 100644 --- a/Canon.Tests/LexicalParserTests/LexicalFileTests.cs +++ b/Canon.Tests/LexicalParserTests/LexicalFileTests.cs @@ -1,149 +1,16 @@ -using System.Text.RegularExpressions; -using Canon.Core.Enums; +using Canon.Core.Enums; using Canon.Core.Exceptions; using Canon.Core.LexicalParser; -using Xunit.Abstractions; using Canon.Tests.Utils; using Canon.Core.Abstractions; +using Xunit.Abstractions; namespace Canon.Tests.LexicalParserTests; -public class LexicalFileTests +public class LexicalFileTests(ITestOutputHelper testOutputHelper) { - private readonly ITestOutputHelper _testOutputHelper; private readonly ILexer _lexer = new Lexer(); - public LexicalFileTests(ITestOutputHelper testOutputHelper) - { - _testOutputHelper = testOutputHelper; - } - - //TODO: 基础的字符串匹配，因此变量名称不能被包含。手写一个存在包含情况的测试文件。 - private static (int, int) FindNthPosition(string pascalProgram, string target, int occurrence) - { - int lineNumber = 0; - (int, int) nthPosition = (0, 0); - int foundCount = 0; - occurrence = occurrence + 1; - - using (StringReader sr = new StringReader(pascalProgram)) - { - string line; - while ((line = sr.ReadLine()) != null) - { - lineNumber++; - int columnNumber = -1; - - // line = Regex.Replace(line, "'[^']*'", "$"); - - while ((columnNumber = line.IndexOf(target, columnNumber + 1, StringComparison.Ordinal)) != -1) - { - foundCount++; - if (foundCount == occurrence) - { - nthPosition = (lineNumber, columnNumber + target.Length); - return nthPosition; - } - } - } - } - - if (nthPosition == (0, 0)) - { - throw new Exception($"'{target}' not found in program."); - } - - return nthPosition; - } - - private void TestLexicalAnalysis(string pascalProgram, List<(string, SemanticTokenType, int)> stringLiterals) - { - var expectedTokens = new List(); - - foreach (var (literal, tokenType, skipCount) in stringLiterals) - { - var (line, column) = FindNthPosition(pascalProgram, literal, skipCount); - switch (tokenType) - { - case SemanticTokenType.Keyword: - expectedTokens.Add(new KeywordSemanticToken - { - LinePos = (uint)line, - CharacterPos = (uint)column, - LiteralValue = literal, - KeywordType = KeywordSemanticToken.GetKeywordTypeByKeyword(literal) - }); - break; - case SemanticTokenType.Identifier: - expectedTokens.Add(new IdentifierSemanticToken - { - LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal - }); - break; - case SemanticTokenType.Delimiter: - if (DelimiterSemanticToken.TryParse((uint)line, (uint)column, new LinkedListNode(literal[0]), - out var delimiterToken)) - { - if (delimiterToken != null) - { - expectedTokens.Add(delimiterToken); - } - } - - break; - case SemanticTokenType.Operator: - expectedTokens.Add(new OperatorSemanticToken - { - LinePos = (uint)line, - CharacterPos = (uint)column, - LiteralValue = literal, - OperatorType = OperatorSemanticToken.GetOperatorTypeByOperator(literal) - }); - break; - case SemanticTokenType.Character: - expectedTokens.Add(new CharacterSemanticToken - { - LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal - }); - break; - case SemanticTokenType.Number: - expectedTokens.Add(new NumberSemanticToken - { - LinePos = (uint)line, - CharacterPos = (uint)column, - LiteralValue = literal, - NumberType = NumberType.Integer - }); - break; - } - } - - expectedTokens = expectedTokens.OrderBy(token => token.LinePos).ThenBy(token => token.CharacterPos).ToList(); - expectedTokens = expectedTokens.Select(token => - token is CharacterSemanticToken characterToken && characterToken.LiteralValue == "hello, world!" - ? new CharacterSemanticToken - { - LinePos = characterToken.LinePos, - CharacterPos = characterToken.CharacterPos + 1, - LiteralValue = characterToken.LiteralValue - } - : token).ToList(); - - IEnumerable tokensEnumerable = _lexer.Tokenize(new StringSourceReader(pascalProgram)); - List tokens = tokensEnumerable.ToList(); - - var actualTokens = tokens; - for (int i = 0; i < expectedTokens.Count; i++) - { - _testOutputHelper.WriteLine($"Expect: {expectedTokens[i]}"); - _testOutputHelper.WriteLine($"Actual: {actualTokens[i]}"); - _testOutputHelper.WriteLine("----"); - // Assert.Equal(expectedTokens[i], actualTokens[i]); - } - - Assert.Equal(expectedTokens, actualTokens); - } - [Fact] public void TestLexicalAnalysisFirst() { @@ -157,30 +24,30 @@ public class LexicalFileTests end. """; - var stringLiterals = new List<(string, SemanticTokenType, int)> - { - ("program", SemanticTokenType.Keyword, 0), - ("HelloWorld", SemanticTokenType.Identifier, 0), - (";", SemanticTokenType.Delimiter, 0), - ("var", SemanticTokenType.Keyword, 0), - ("message", SemanticTokenType.Identifier, 0), - (":", SemanticTokenType.Delimiter, 0), - ("string", SemanticTokenType.Identifier, 0), - (";", SemanticTokenType.Delimiter, 1), - ("begin", SemanticTokenType.Keyword, 0), - ("message", SemanticTokenType.Identifier, 1), - (":=", SemanticTokenType.Operator, 0), - ("hello, world!", SemanticTokenType.Character, 0), - (";", SemanticTokenType.Delimiter, 2), - ("writeln", SemanticTokenType.Identifier, 0), - ("(", SemanticTokenType.Delimiter, 0), - ("message", SemanticTokenType.Identifier, 2), - (")", SemanticTokenType.Delimiter, 0), - (";", SemanticTokenType.Delimiter, 3), - ("end", SemanticTokenType.Keyword, 0), - (".", SemanticTokenType.Delimiter, 0) - }; - TestLexicalAnalysis(pascalProgram, stringLiterals); + IEnumerable tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram)); + ValidateSemanticTokens(tokens, [ + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Character, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter, + SemanticTokenType.End + ]); } [Fact] @@ -196,30 +63,30 @@ public class LexicalFileTests end. """; - var stringLiterals = new List<(string, SemanticTokenType, int)> - { - ("program", SemanticTokenType.Keyword, 0), - ("main", SemanticTokenType.Identifier, 0), - (";", SemanticTokenType.Delimiter, 0), - ("var", SemanticTokenType.Keyword, 0), - ("ab", SemanticTokenType.Identifier, 0), - (":", SemanticTokenType.Delimiter, 0), - ("integer", SemanticTokenType.Keyword, 0), - (";", SemanticTokenType.Delimiter, 1), - ("begin", SemanticTokenType.Keyword, 0), - ("ab", SemanticTokenType.Identifier, 1), - (":=", SemanticTokenType.Operator, 0), - ("3", SemanticTokenType.Number, 0), - (";", SemanticTokenType.Delimiter, 2), - ("write", SemanticTokenType.Identifier, 0), - ("(", SemanticTokenType.Delimiter, 0), - ("ab", SemanticTokenType.Identifier, 2), - (")", SemanticTokenType.Delimiter, 0), - (";", SemanticTokenType.Delimiter, 3), - ("end", SemanticTokenType.Keyword, 0), - (".", SemanticTokenType.Delimiter, 0) - }; - TestLexicalAnalysis(pascalProgram, stringLiterals); + IEnumerable tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram)); + + ValidateSemanticTokens(tokens, [ + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Number, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter + ]); } //带注释的测试 @@ -239,42 +106,74 @@ public class LexicalFileTests end. """; - var stringLiterals = new List<(string, SemanticTokenType, int)> - { - ("program", SemanticTokenType.Keyword, 0), - ("main", SemanticTokenType.Identifier, 0), - (";", SemanticTokenType.Delimiter, 0), - ("var", SemanticTokenType.Keyword, 0), - ("ab", SemanticTokenType.Identifier, 0), - (",", SemanticTokenType.Delimiter, 0), - ("ba", SemanticTokenType.Identifier, 0), - (":", SemanticTokenType.Delimiter, 0), - ("integer", SemanticTokenType.Keyword, 0), - (";", SemanticTokenType.Delimiter, 1), - ("begin", SemanticTokenType.Keyword, 0), - ("ab", SemanticTokenType.Identifier, 1), - (":=", SemanticTokenType.Operator, 0), - ("3", SemanticTokenType.Number, 0), - (";", SemanticTokenType.Delimiter, 2), - ("ba", SemanticTokenType.Identifier, 1), - (":=", SemanticTokenType.Operator, 1), - ("5", SemanticTokenType.Number, 0), - (";", SemanticTokenType.Delimiter, 3), - ("ab", SemanticTokenType.Identifier, 2), - (":=", SemanticTokenType.Operator, 2), - ("5", SemanticTokenType.Number, 1), - (";", SemanticTokenType.Delimiter, 4), - ("write", SemanticTokenType.Identifier, 0), - ("(", SemanticTokenType.Delimiter, 0), - ("ab", SemanticTokenType.Identifier, 3), - ("+", SemanticTokenType.Operator, 0), - ("ba", SemanticTokenType.Identifier, 2), - (")", SemanticTokenType.Delimiter, 0), - (";", SemanticTokenType.Delimiter, 5), - ("end", SemanticTokenType.Keyword, 0), - (".", SemanticTokenType.Delimiter, 0) - }; - TestLexicalAnalysis(pascalProgram, stringLiterals); + IEnumerable tokens = _lexer.Tokenize(new StringSourceReader(pascalProgram)); + + ValidateSemanticTokens(tokens, [ + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Number, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Number, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Number, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Identifier, + SemanticTokenType.Operator, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter, + SemanticTokenType.End + ]); + } + + [Fact] + public void ReuseTest() + { + const string program1 = """ + program main; + begin + end. + """; + IEnumerable tokens = _lexer.Tokenize(new StringSourceReader(program1)); + + ValidateSemanticTokens(tokens, [ + SemanticTokenType.Keyword, + SemanticTokenType.Identifier, + SemanticTokenType.Delimiter, + SemanticTokenType.Keyword, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter + ]); + + const string test = "program begin end."; + + tokens = _lexer.Tokenize(new StringSourceReader(test)); + + ValidateSemanticTokens(tokens, [ + SemanticTokenType.Keyword, + SemanticTokenType.Keyword, + SemanticTokenType.Keyword, + SemanticTokenType.Delimiter + ]); } [Fact] @@ -291,7 +190,7 @@ public class LexicalFileTests """; var ex = Assert.Throws(() => _lexer.Tokenize(new StringSourceReader(pascalProgram)).ToList()); //打印exception信息 - _testOutputHelper.WriteLine(ex.ToString()); + testOutputHelper.WriteLine(ex.ToString()); Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType); Assert.Equal((uint)7, ex.Line); Assert.Equal((uint)5, ex.CharPosition); @@ -307,7 +206,7 @@ public class LexicalFileTests program CommentNotClosed; """; var ex = Assert.Throws(() => _lexer.Tokenize(new StringSourceReader(pascalProgram)).ToList()); - _testOutputHelper.WriteLine(ex.ToString()); + testOutputHelper.WriteLine(ex.ToString()); Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType); Assert.Equal((uint)4, ex.Line); Assert.Equal((uint)26, ex.CharPosition); @@ -410,4 +309,13 @@ public class LexicalFileTests List tokens = tokensEnumerable.ToList(); Assert.NotNull(tokens); } + + private static void ValidateSemanticTokens(IEnumerable actualTokens, + IEnumerable expectedTypes) + { + foreach ((SemanticTokenType type, SemanticToken token) in expectedTypes.Zip(actualTokens)) + { + Assert.Equal(type, token.TokenType); + } + } }