diff --git a/Canon.Core/LexicalParser/Lexer.cs b/Canon.Core/LexicalParser/Lexer.cs index 31031b7..3d618d3 100644 --- a/Canon.Core/LexicalParser/Lexer.cs +++ b/Canon.Core/LexicalParser/Lexer.cs @@ -1,95 +1,92 @@ - +namespace Canon.Core.LexicalParser; -namespace Canon.Core.LexicalParser +public class Lexer { - public class Lexer + private readonly LinkedList _source; + private LinkedListNode? _currentNode; + private uint _line = 1; + private uint _charPosition; + private readonly List _tokens = []; + + public Lexer(string source) { - private readonly LinkedList source; - private LinkedListNode? currentNode; - private uint line = 1; - private uint charPosition = 0; - private List tokens = new List(); + // 将字符串转换为LinkedList + _source = new LinkedList(source); + _currentNode = _source.First; + } - public Lexer(string source) + public List Tokenize() + { + while (_currentNode != null) { - // 将字符串转换为LinkedList - this.source = new LinkedList(source); - currentNode = this.source.First; - } + _charPosition = 0; // 重置字符位置 + SkipWhitespace(); - public List Tokenize() - { - while (currentNode != null) + if (_currentNode == null) break; // 如果跳过空格后到达了末尾,则退出循环 + + SemanticToken? token = null; + + // 尝试解析各种类型的词法单元 + if (DelimiterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var delimiterToken)) { - charPosition = 0; // 重置字符位置 - SkipWhitespace(); - - if (currentNode == null) break; // 如果跳过空格后到达了末尾,则退出循环 - - SemanticToken? token = null; - - // 尝试解析各种类型的词法单元 - if (DelimiterSemanticToken.TryParse(line, charPosition, currentNode, out var delimiterToken)) - { - token = delimiterToken; - } - else if (CharacterSemanticToken.TryParse(line, charPosition, currentNode, out var characterToken)) - { - token = characterToken; - } - else if (KeywordSemanticToken.TryParse(line, charPosition, currentNode, out var keywordToken)) - { - token = keywordToken; - } - else if (OperatorSemanticToken.TryParse(line, charPosition, currentNode, out var operatorToken)) - { - token = operatorToken; - } - else if (NumberSemanticToken.TryParse(line, charPosition, currentNode, out var numberToken)) - { - token = numberToken; - } - else if (IdentifierSemanticToken.TryParse(line, charPosition, currentNode, out var identifierToken)) - { - token = identifierToken; - } - - if (token != null) - { - tokens.Add(token); - // 根据词法单元的长度移动currentNode - MoveCurrentNode(token.LiteralValue.Length); - } - else - { - // 未能识别的字符,跳过 - MoveCurrentNode(1); - } + token = delimiterToken; + } + else if (CharacterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var characterToken)) + { + token = characterToken; + } + else if (KeywordSemanticToken.TryParse(_line, _charPosition, _currentNode, out var keywordToken)) + { + token = keywordToken; + } + else if (OperatorSemanticToken.TryParse(_line, _charPosition, _currentNode, out var operatorToken)) + { + token = operatorToken; + } + else if (NumberSemanticToken.TryParse(_line, _charPosition, _currentNode, out var numberToken)) + { + token = numberToken; + } + else if (IdentifierSemanticToken.TryParse(_line, _charPosition, _currentNode, out var identifierToken)) + { + token = identifierToken; } - // tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记 - return tokens; - } - - private void SkipWhitespace() - { - while (currentNode != null && char.IsWhiteSpace(currentNode.Value)) + if (token != null) { - if (currentNode.Value == '\n') - { - line++; - charPosition = 0; - } - currentNode = currentNode.Next; + _tokens.Add(token); + // 根据词法单元的长度移动currentNode + MoveCurrentNode(token.LiteralValue.Length); + } + else + { + // 未能识别的字符,跳过 + MoveCurrentNode(1); } } - private void MoveCurrentNode(int steps) + // tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记 + return _tokens; + } + + private void SkipWhitespace() + { + while (_currentNode != null && char.IsWhiteSpace(_currentNode.Value)) { - for (int i = 0; i < steps && currentNode != null; i++) + if (_currentNode.Value == '\n') { - currentNode = currentNode.Next; + _line++; + _charPosition = 0; } + _currentNode = _currentNode.Next; + } + } + + private void MoveCurrentNode(int steps) + { + for (int i = 0; i < steps && _currentNode != null; i++) + { + _currentNode = _currentNode.Next; } } }