post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refact: 合乎代码规范

Browse Source
This commit is contained in:
jackfiled 2024-03-11 11:36:39 +08:00
parent 9984cee920
commit 46c4a5e5ea
1 changed files with 73 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
Canon.Core/LexicalParser/Lexer.cs
View File

@ -1,95 +1,92 @@

namespace Canon.Core.LexicalParser;
namespace Canon.Core.LexicalParser
public class Lexer
{
public class Lexer
private readonly LinkedList<char> _source;
private LinkedListNode<char>? _currentNode;
private uint _line = 1;
private uint _charPosition;
private readonly List<SemanticToken> _tokens = [];
public Lexer(string source)
{
private readonly LinkedList<char> source;
private LinkedListNode<char>? currentNode;
private uint line = 1;
private uint charPosition = 0;
private List<SemanticToken> tokens = new List<SemanticToken>();
// 将字符串转换为LinkedList<char>
_source = new LinkedList<char>(source);
_currentNode = _source.First;
}
public Lexer(string source)
public List<SemanticToken> Tokenize()
{
while (_currentNode != null)
{
// 将字符串转换为LinkedList<char>
this.source = new LinkedList<char>(source);
currentNode = this.source.First;
}
_charPosition = 0; // 重置字符位置
SkipWhitespace();
public List<SemanticToken> Tokenize()
{
while (currentNode != null)
if (_currentNode == null) break; // 如果跳过空格后到达了末尾，则退出循环
SemanticToken? token = null;
// 尝试解析各种类型的词法单元
if (DelimiterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var delimiterToken))
{
charPosition = 0; // 重置字符位置
SkipWhitespace();
if (currentNode == null) break; // 如果跳过空格后到达了末尾，则退出循环
SemanticToken? token = null;
// 尝试解析各种类型的词法单元
if (DelimiterSemanticToken.TryParse(line, charPosition, currentNode, out var delimiterToken))
{
token = delimiterToken;
}
else if (CharacterSemanticToken.TryParse(line, charPosition, currentNode, out var characterToken))
{
token = characterToken;
}
else if (KeywordSemanticToken.TryParse(line, charPosition, currentNode, out var keywordToken))
{
token = keywordToken;
}
else if (OperatorSemanticToken.TryParse(line, charPosition, currentNode, out var operatorToken))
{
token = operatorToken;
}
else if (NumberSemanticToken.TryParse(line, charPosition, currentNode, out var numberToken))
{
token = numberToken;
}
else if (IdentifierSemanticToken.TryParse(line, charPosition, currentNode, out var identifierToken))
{
token = identifierToken;
}
if (token != null)
{
tokens.Add(token);
// 根据词法单元的长度移动currentNode
MoveCurrentNode(token.LiteralValue.Length);
}
else
{
// 未能识别的字符，跳过
MoveCurrentNode(1);
}
token = delimiterToken;
}
else if (CharacterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var characterToken))
{
token = characterToken;
}
else if (KeywordSemanticToken.TryParse(_line, _charPosition, _currentNode, out var keywordToken))
{
token = keywordToken;
}
else if (OperatorSemanticToken.TryParse(_line, _charPosition, _currentNode, out var operatorToken))
{
token = operatorToken;
}
else if (NumberSemanticToken.TryParse(_line, _charPosition, _currentNode, out var numberToken))
{
token = numberToken;
}
else if (IdentifierSemanticToken.TryParse(_line, _charPosition, _currentNode, out var identifierToken))
{
token = identifierToken;
}
// tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记
return tokens;
}
private void SkipWhitespace()
{
while (currentNode != null && char.IsWhiteSpace(currentNode.Value))
if (token != null)
{
if (currentNode.Value == '\n')
{
line++;
charPosition = 0;
}
currentNode = currentNode.Next;
_tokens.Add(token);
// 根据词法单元的长度移动currentNode
MoveCurrentNode(token.LiteralValue.Length);
}
else
{
// 未能识别的字符，跳过
MoveCurrentNode(1);
}
}
private void MoveCurrentNode(int steps)
// tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记
return _tokens;
}
private void SkipWhitespace()
{
while (_currentNode != null && char.IsWhiteSpace(_currentNode.Value))
{
for (int i = 0; i < steps && currentNode != null; i++)
if (_currentNode.Value == '\n')
{
currentNode = currentNode.Next;
_line++;
_charPosition = 0;
}
_currentNode = _currentNode.Next;
}
}
private void MoveCurrentNode(int steps)
{
for (int i = 0; i < steps && _currentNode != null; i++)
{
_currentNode = _currentNode.Next;
}
}
}