refact: 合乎代码规范
This commit is contained in:
parent
9984cee920
commit
46c4a5e5ea
|
@ -1,95 +1,92 @@
|
||||||
|
namespace Canon.Core.LexicalParser;
|
||||||
|
|
||||||
namespace Canon.Core.LexicalParser
|
public class Lexer
|
||||||
{
|
{
|
||||||
public class Lexer
|
private readonly LinkedList<char> _source;
|
||||||
|
private LinkedListNode<char>? _currentNode;
|
||||||
|
private uint _line = 1;
|
||||||
|
private uint _charPosition;
|
||||||
|
private readonly List<SemanticToken> _tokens = [];
|
||||||
|
|
||||||
|
public Lexer(string source)
|
||||||
{
|
{
|
||||||
private readonly LinkedList<char> source;
|
// 将字符串转换为LinkedList<char>
|
||||||
private LinkedListNode<char>? currentNode;
|
_source = new LinkedList<char>(source);
|
||||||
private uint line = 1;
|
_currentNode = _source.First;
|
||||||
private uint charPosition = 0;
|
}
|
||||||
private List<SemanticToken> tokens = new List<SemanticToken>();
|
|
||||||
|
|
||||||
public Lexer(string source)
|
public List<SemanticToken> Tokenize()
|
||||||
|
{
|
||||||
|
while (_currentNode != null)
|
||||||
{
|
{
|
||||||
// 将字符串转换为LinkedList<char>
|
_charPosition = 0; // 重置字符位置
|
||||||
this.source = new LinkedList<char>(source);
|
SkipWhitespace();
|
||||||
currentNode = this.source.First;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<SemanticToken> Tokenize()
|
if (_currentNode == null) break; // 如果跳过空格后到达了末尾,则退出循环
|
||||||
{
|
|
||||||
while (currentNode != null)
|
SemanticToken? token = null;
|
||||||
|
|
||||||
|
// 尝试解析各种类型的词法单元
|
||||||
|
if (DelimiterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var delimiterToken))
|
||||||
{
|
{
|
||||||
charPosition = 0; // 重置字符位置
|
token = delimiterToken;
|
||||||
SkipWhitespace();
|
}
|
||||||
|
else if (CharacterSemanticToken.TryParse(_line, _charPosition, _currentNode, out var characterToken))
|
||||||
if (currentNode == null) break; // 如果跳过空格后到达了末尾,则退出循环
|
{
|
||||||
|
token = characterToken;
|
||||||
SemanticToken? token = null;
|
}
|
||||||
|
else if (KeywordSemanticToken.TryParse(_line, _charPosition, _currentNode, out var keywordToken))
|
||||||
// 尝试解析各种类型的词法单元
|
{
|
||||||
if (DelimiterSemanticToken.TryParse(line, charPosition, currentNode, out var delimiterToken))
|
token = keywordToken;
|
||||||
{
|
}
|
||||||
token = delimiterToken;
|
else if (OperatorSemanticToken.TryParse(_line, _charPosition, _currentNode, out var operatorToken))
|
||||||
}
|
{
|
||||||
else if (CharacterSemanticToken.TryParse(line, charPosition, currentNode, out var characterToken))
|
token = operatorToken;
|
||||||
{
|
}
|
||||||
token = characterToken;
|
else if (NumberSemanticToken.TryParse(_line, _charPosition, _currentNode, out var numberToken))
|
||||||
}
|
{
|
||||||
else if (KeywordSemanticToken.TryParse(line, charPosition, currentNode, out var keywordToken))
|
token = numberToken;
|
||||||
{
|
}
|
||||||
token = keywordToken;
|
else if (IdentifierSemanticToken.TryParse(_line, _charPosition, _currentNode, out var identifierToken))
|
||||||
}
|
{
|
||||||
else if (OperatorSemanticToken.TryParse(line, charPosition, currentNode, out var operatorToken))
|
token = identifierToken;
|
||||||
{
|
|
||||||
token = operatorToken;
|
|
||||||
}
|
|
||||||
else if (NumberSemanticToken.TryParse(line, charPosition, currentNode, out var numberToken))
|
|
||||||
{
|
|
||||||
token = numberToken;
|
|
||||||
}
|
|
||||||
else if (IdentifierSemanticToken.TryParse(line, charPosition, currentNode, out var identifierToken))
|
|
||||||
{
|
|
||||||
token = identifierToken;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (token != null)
|
|
||||||
{
|
|
||||||
tokens.Add(token);
|
|
||||||
// 根据词法单元的长度移动currentNode
|
|
||||||
MoveCurrentNode(token.LiteralValue.Length);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// 未能识别的字符,跳过
|
|
||||||
MoveCurrentNode(1);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记
|
if (token != null)
|
||||||
return tokens;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SkipWhitespace()
|
|
||||||
{
|
|
||||||
while (currentNode != null && char.IsWhiteSpace(currentNode.Value))
|
|
||||||
{
|
{
|
||||||
if (currentNode.Value == '\n')
|
_tokens.Add(token);
|
||||||
{
|
// 根据词法单元的长度移动currentNode
|
||||||
line++;
|
MoveCurrentNode(token.LiteralValue.Length);
|
||||||
charPosition = 0;
|
}
|
||||||
}
|
else
|
||||||
currentNode = currentNode.Next;
|
{
|
||||||
|
// 未能识别的字符,跳过
|
||||||
|
MoveCurrentNode(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void MoveCurrentNode(int steps)
|
// tokens.Add(new EOFToken(line, charPosition)); // 添加EOF标记
|
||||||
|
return _tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SkipWhitespace()
|
||||||
|
{
|
||||||
|
while (_currentNode != null && char.IsWhiteSpace(_currentNode.Value))
|
||||||
{
|
{
|
||||||
for (int i = 0; i < steps && currentNode != null; i++)
|
if (_currentNode.Value == '\n')
|
||||||
{
|
{
|
||||||
currentNode = currentNode.Next;
|
_line++;
|
||||||
|
_charPosition = 0;
|
||||||
}
|
}
|
||||||
|
_currentNode = _currentNode.Next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void MoveCurrentNode(int steps)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < steps && _currentNode != null; i++)
|
||||||
|
{
|
||||||
|
_currentNode = _currentNode.Next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user