post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: uninitialized property _reader in Lexer.

Browse Source
This commit is contained in:
jackfiled 2025-02-19 16:29:33 +08:00
parent ad44a37b3c
commit 58deabb023
2 changed files with 671 additions and 662 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

664
Canon.Core/LexicalParser/Lexer.cs
View File

@ -1,673 +1,13 @@
using System.Text;
using Canon.Core.Abstractions;
using Canon.Core.Enums;
using Canon.Core.Exceptions;
namespace Canon.Core.LexicalParser;
public class Lexer : ILexer
{
// 记录token
private SemanticToken? _semanticToken;
private readonly StringBuilder _tokenBuilder = new();
private List<SemanticToken> _tokens = [];
// 状态机
private StateType _state = StateType.Start;
private char _ch;
private bool _finish;
// 文件读取
private ISourceReader _reader;
private uint _line = 1;
private uint _chPos;
public IEnumerable<SemanticToken> Tokenize(ISourceReader reader)
{
_reader = reader;
_tokens = [];
_state = StateType.Start;
LexerStateMachine machine = new(reader);
while (_state != StateType.Done)
{
switch (_state)
{
case StateType.Start:
HandleStartState();
break;
case StateType.Comment:
if (_ch == '{')
{
HandleCommentStateBig();
}
else if (_ch == '*')
{
HandleCommentStateSmall();
}
else
{
HandleCommentSingleLine();
}
break;
case StateType.Num:
HandleNumState();
break;
case StateType.Word:
HandleWordState();
break;
case StateType.Delimiter:
HandleDelimiterState();
break;
case StateType.Operator:
HandleOperatorState();
break;
case StateType.BreakPoint:
while (LexRules.IsBreakPoint(_ch))
{
GetChar();
}
Retract();
_state = StateType.Start;
break;
case StateType.Unknown:
throw new LexemeException(LexemeErrorType.UnknownCharacterOrString, _line, _chPos,
"Illegal lexeme.");
case StateType.Done:
break;
}
}
_tokens.Add(SemanticToken.End);
return _tokens;
}
private void HandleStartState()
{
// 初始化
ResetTokenBuilder();
// 读取首个字符
GetChar();
if (_finish)
{
_state = StateType.Done;
return;
}
// 根据首个字符判断可能的情况
if (_ch == '{') // 以 “{” 开头，为注释
{
_state = StateType.Comment;
}
else if (_ch == '(')
{
char nextChar = PeekNextChar();
if (nextChar == '*')
{
GetChar();
_state = StateType.Comment;
}
else
{
_state = StateType.Delimiter;
}
}
else if (_ch == '/')
{
char nextChar = PeekNextChar();
if (nextChar == '/')
{
GetChar();
_state = StateType.Comment;
}
else
{
_state = StateType.Operator;
}
}
else if (_ch == '.') // 以 “.” 开头，可能是数字或分隔符
{
char next = PeekNextChar();
if (next is >= '0' and <= '9')
{
_state = StateType.Num;
}
else
{
_state = StateType.Delimiter;
}
}
else if (LexRules.IsLetter(_ch)) // 以字母开头，为关键字或标识符
{
_state = StateType.Word;
}
else if (LexRules.IsDigit(_ch) || _ch == '$') // 以数字或 “$” 开头，为数值
{
_state = StateType.Num;
}
else if (LexRules.IsDelimiter(_ch)) // 为分隔符
{
_state = StateType.Delimiter;
}
else if (LexRules.IsOperator(_ch)) // 为运算符
{
_state = StateType.Operator;
}
else if (LexRules.IsBreakPoint(_ch))
{
_state = StateType.BreakPoint;
}
else
{
_state = StateType.Unknown;
}
}
private void HandleCommentStateBig()
{
while (_ch != '}')
{
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
}
_state = StateType.Start;
}
private void HandleCommentStateSmall()
{
bool commentClosed = false;
while (!commentClosed)
{
GetChar();
while (_ch != '*')
{
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
}
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
if (_ch == ')') commentClosed = true;
}
_state = StateType.Start;
}
private void HandleCommentSingleLine()
{
while (_ch != '\n')
{
GetChar();
}
_state = StateType.Start;
}
private void HandleWordState()
{
while (LexRules.IsDigit(_ch) || LexRules.IsLetter(_ch))
{
Cat();
GetChar();
}
Retract();
string tokenString = GetCurrentTokenString();
if (LexRules.GetKeywordTypeByKeywprd(tokenString, out KeywordType keywordType))
{
_semanticToken = LexemeFactory.MakeToken(keywordType, tokenString, _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.Identifier, tokenString, _line, _chPos);
}
AddToTokens(_semanticToken);
_state = StateType.Start;
}
private void HandleNumState()
{
NumberType numberType = NumberType.Integer;
// 十六进制
if (_ch == '$')
{
ProcessHex();
numberType = NumberType.Hex;
}
// 非十六进制
else if (LexRules.IsDigit(_ch) || _ch == '.')
{
while (!NumberShouldBreak())
{
// 含小数部分
if (_ch == '.')
{
// 检查是否是符号 “..”
char next = PeekNextChar();
if (next == '.')
{
Retract();
_state = StateType.Delimiter;
break;
}
// 不是符号 “..”,进入小数点后的判断
Cat(); // 记录“.”
// “.”后不应为空，至少应该有一位小数
GetChar();
if (NumberShouldBreak())
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal numbers!");
}
// 读取小数点后的数字
while (!NumberShouldBreak())
{
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else if (_ch == 'e' || _ch == 'E')
{
ProcessE();
break;
}
else if (NumberShouldBreak())
{
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal number.");
}
}
numberType = NumberType.Real;
break;
}
// 不含小数部分，含科学计数法
if (_ch == 'e' || _ch == 'E')
{
ProcessE();
numberType = NumberType.Real;
break;
}
// 暂时为整数
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else if (NumberShouldBreak())
{
numberType = NumberType.Integer;
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
}
_semanticToken = LexemeFactory.MakeToken(numberType, GetCurrentTokenString(),
_line, _chPos);
AddToTokens(_semanticToken);
_state = StateType.Start;
}
private void ProcessHex()
{
Cat();
GetChar();
while (!NumberShouldBreak())
{
// 假设IsHexDigit方法能够识别十六进制数字
if (LexRules.IsHexDigit(_ch))
{
Cat();
GetChar();
}
else if (NumberShouldBreak())
{
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal hex numbers!");
}
}
}
private void ProcessE()
{
Cat();
GetChar();
if (LexRules.IsDigit(_ch) || _ch == '+' || _ch == '-')
{
Cat();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
// 读取e后的数字
GetChar();
while (!NumberShouldBreak())
{
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
}
bool NumberShouldBreak()
{
if (_ch == ' ' || _ch == '\n' || _ch == '\t' || _ch == '\r' || (LexRules.IsDelimiter(_ch) && _ch != '.') ||
LexRules.IsOperator(_ch) || _finish)
{
Retract();
return true;
}
return false;
}
private bool IsDot()
{
if (_tokens.Count != 0)
{
SemanticToken tokenBefore = _tokens.Last();
if (tokenBefore.TokenType == SemanticTokenType.Identifier) return true;
}
return false;
}
private void HandleDelimiterState()
{
Cat();
switch (_ch)
{
case '.':
{
GetChar();
if (_ch == '.')
{
Cat();
_semanticToken = LexemeFactory.MakeToken(DelimiterType.DoubleDots, "..", _line, _chPos);
break;
}
Retract();
if (IsDot())
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Dot, ".", _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Period, ".", _line, _chPos);
}
}
break;
case '\'':
{
// 重置_token，准备收集字符串内容
ResetTokenBuilder();
GetChar(); // 移动到下一个字符，即字符串的第一个字符
while (_ch != '\'' && _ch != '\"')
{
Cat(); // 收集字符
GetChar(); // 移动到下一个字符
if (_ch == '\n' || _finish)
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos,
"The String is not closed.");
}
}
string currentString = GetCurrentTokenString();
if (currentString.Length > 1)
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.String,
currentString, _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.Character,
currentString, _line, _chPos);
}
ResetTokenBuilder();
if (!(_ch == '\'' || _ch == '\"'))
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos,
"The String is not closed.");
}
}
break;
case ',':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Comma, ",", _line, _chPos);
break;
case ':':
char nextChar = PeekNextChar();
if (nextChar == '=')
{
GetChar();
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Assign, ":=", _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Colon, ":", _line, _chPos);
}
break;
case ';':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Semicolon, ";", _line, _chPos);
break;
case '(':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.LeftParenthesis, "(", _line, _chPos);
break;
case ')':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.RightParenthesis, ")", _line, _chPos);
break;
case '[':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.LeftSquareBracket, "[", _line, _chPos);
break;
case ']':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.RightSquareBracket, "]", _line, _chPos);
break;
}
if (_semanticToken is null)
{
throw new InvalidOperationException();
}
_tokens.Add(_semanticToken);
_state = StateType.Start;
}
private void HandleOperatorState()
{
switch (_ch)
{
case '+': // 识别 +
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Plus, "+", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '-': // 识别 -
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Minus, "-", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '*': // 识别 *
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Multiply, "*", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '/': // 识别 /
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Divide, "/", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '=':
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Equal, "=", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '<':
Cat();
GetChar();
if (_ch == '=')
{
// 识别 <=
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.LessEqual, "<=", _line, _chPos);
AddToTokens(_semanticToken);
}
else if (_ch == '>')
{
// 识别 <>
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.NotEqual, ">", _line, _chPos);
AddToTokens(_semanticToken);
}
else
{
// 识别 <
Retract();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Less, "<", _line, _chPos);
AddToTokens(_semanticToken);
}
break;
case '>':
Cat();
GetChar();
if (_ch == '=')
{
// 识别 >=
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.GreaterEqual, ">=", _line, _chPos);
AddToTokens(_semanticToken);
}
else
{
// 识别 >
Retract();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Greater, ">", _line, _chPos);
AddToTokens(_semanticToken);
}
break;
default:
throw new LexemeException(LexemeErrorType.UnknownCharacterOrString, _line, _chPos, "Illegal lexeme.");
}
_state = StateType.Start;
}
private void AddToTokens(SemanticToken semanticToken)
{
_tokens.Add(semanticToken);
}
private void Cat()
{
_tokenBuilder.Append(_ch); // 使用StringBuilder追加字符
}
private string GetCurrentTokenString()
{
return _tokenBuilder.ToString(); // 从StringBuilder获取当前记号的字符串
}
private void ResetTokenBuilder()
{
_tokenBuilder.Clear(); // 清空StringBuilder以复用
}
private char PeekNextChar()
{
// 确认下一个位置是否仍在buffer的范围内
if (_reader.TryPeekChar(out char? c))
{
return c.Value;
}
else
{
return char.MinValue;
}
}
void GetChar()
{
if (_finish)
{
return;
}
_finish = !_reader.MoveNext();
if (_finish)
{
_ch = char.MinValue;
return;
}
_ch = _reader.Current;
_line = _reader.Line;
_chPos = _reader.Pos;
}
void Retract()
{
_reader.Retract();
return machine.Run();
}
}

669
Canon.Core/LexicalParser/LexerStateMachine.cs Normal file
View File

@ -0,0 +1,669 @@
using System.Text;
using Canon.Core.Abstractions;
using Canon.Core.Enums;
using Canon.Core.Exceptions;
namespace Canon.Core.LexicalParser;
public sealed class LexerStateMachine(ISourceReader reader)
{
// 记录token
private SemanticToken? _semanticToken;
private readonly StringBuilder _tokenBuilder = new();
private readonly List<SemanticToken> _tokens = [];
// 状态机
private StateType _state = StateType.Start;
private char _ch;
private bool _finish;
// 文件读取
private uint _line = 1;
private uint _chPos;
public IEnumerable<SemanticToken> Run()
{
while (_state != StateType.Done)
{
switch (_state)
{
case StateType.Start:
HandleStartState();
break;
case StateType.Comment:
if (_ch == '{')
{
HandleCommentStateBig();
}
else if (_ch == '*')
{
HandleCommentStateSmall();
}
else
{
HandleCommentSingleLine();
}
break;
case StateType.Num:
HandleNumState();
break;
case StateType.Word:
HandleWordState();
break;
case StateType.Delimiter:
HandleDelimiterState();
break;
case StateType.Operator:
HandleOperatorState();
break;
case StateType.BreakPoint:
while (LexRules.IsBreakPoint(_ch))
{
GetChar();
}
Retract();
_state = StateType.Start;
break;
case StateType.Unknown:
throw new LexemeException(LexemeErrorType.UnknownCharacterOrString, _line, _chPos,
"Illegal lexeme.");
case StateType.Done:
break;
}
}
_tokens.Add(SemanticToken.End);
return _tokens;
}
private void HandleStartState()
{
// 初始化
ResetTokenBuilder();
// 读取首个字符
GetChar();
if (_finish)
{
_state = StateType.Done;
return;
}
// 根据首个字符判断可能的情况
if (_ch == '{') // 以 “{” 开头，为注释
{
_state = StateType.Comment;
}
else if (_ch == '(')
{
char nextChar = PeekNextChar();
if (nextChar == '*')
{
GetChar();
_state = StateType.Comment;
}
else
{
_state = StateType.Delimiter;
}
}
else if (_ch == '/')
{
char nextChar = PeekNextChar();
if (nextChar == '/')
{
GetChar();
_state = StateType.Comment;
}
else
{
_state = StateType.Operator;
}
}
else if (_ch == '.') // 以 “.” 开头，可能是数字或分隔符
{
char next = PeekNextChar();
if (next is >= '0' and <= '9')
{
_state = StateType.Num;
}
else
{
_state = StateType.Delimiter;
}
}
else if (LexRules.IsLetter(_ch)) // 以字母开头，为关键字或标识符
{
_state = StateType.Word;
}
else if (LexRules.IsDigit(_ch) || _ch == '$') // 以数字或 “$” 开头，为数值
{
_state = StateType.Num;
}
else if (LexRules.IsDelimiter(_ch)) // 为分隔符
{
_state = StateType.Delimiter;
}
else if (LexRules.IsOperator(_ch)) // 为运算符
{
_state = StateType.Operator;
}
else if (LexRules.IsBreakPoint(_ch))
{
_state = StateType.BreakPoint;
}
else
{
_state = StateType.Unknown;
}
}
private void HandleCommentStateBig()
{
while (_ch != '}')
{
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
}
_state = StateType.Start;
}
private void HandleCommentStateSmall()
{
bool commentClosed = false;
while (!commentClosed)
{
GetChar();
while (_ch != '*')
{
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
}
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos,
"The comment is not closed.");
}
if (_ch == ')') commentClosed = true;
}
_state = StateType.Start;
}
private void HandleCommentSingleLine()
{
while (_ch != '\n')
{
GetChar();
}
_state = StateType.Start;
}
private void HandleWordState()
{
while (LexRules.IsDigit(_ch) || LexRules.IsLetter(_ch))
{
Cat();
GetChar();
}
Retract();
string tokenString = GetCurrentTokenString();
if (LexRules.GetKeywordTypeByKeywprd(tokenString, out KeywordType keywordType))
{
_semanticToken = LexemeFactory.MakeToken(keywordType, tokenString, _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.Identifier, tokenString, _line, _chPos);
}
AddToTokens(_semanticToken);
_state = StateType.Start;
}
private void HandleNumState()
{
NumberType numberType = NumberType.Integer;
// 十六进制
if (_ch == '$')
{
ProcessHex();
numberType = NumberType.Hex;
}
// 非十六进制
else if (LexRules.IsDigit(_ch) || _ch == '.')
{
while (!NumberShouldBreak())
{
// 含小数部分
if (_ch == '.')
{
// 检查是否是符号 “..”
char next = PeekNextChar();
if (next == '.')
{
Retract();
_state = StateType.Delimiter;
break;
}
// 不是符号 “..”,进入小数点后的判断
Cat(); // 记录“.”
// “.”后不应为空，至少应该有一位小数
GetChar();
if (NumberShouldBreak())
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal numbers!");
}
// 读取小数点后的数字
while (!NumberShouldBreak())
{
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else if (_ch == 'e' || _ch == 'E')
{
ProcessE();
break;
}
else if (NumberShouldBreak())
{
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal number.");
}
}
numberType = NumberType.Real;
break;
}
// 不含小数部分，含科学计数法
if (_ch == 'e' || _ch == 'E')
{
ProcessE();
numberType = NumberType.Real;
break;
}
// 暂时为整数
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else if (NumberShouldBreak())
{
numberType = NumberType.Integer;
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
}
_semanticToken = LexemeFactory.MakeToken(numberType, GetCurrentTokenString(),
_line, _chPos);
AddToTokens(_semanticToken);
_state = StateType.Start;
}
private void ProcessHex()
{
Cat();
GetChar();
while (!NumberShouldBreak())
{
// 假设IsHexDigit方法能够识别十六进制数字
if (LexRules.IsHexDigit(_ch))
{
Cat();
GetChar();
}
else if (NumberShouldBreak())
{
break;
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos,
"Illegal hex numbers!");
}
}
}
private void ProcessE()
{
Cat();
GetChar();
if (LexRules.IsDigit(_ch) || _ch == '+' || _ch == '-')
{
Cat();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
// 读取e后的数字
GetChar();
while (!NumberShouldBreak())
{
if (LexRules.IsDigit(_ch))
{
Cat();
GetChar();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
}
bool NumberShouldBreak()
{
if (_ch == ' ' || _ch == '\n' || _ch == '\t' || _ch == '\r' || (LexRules.IsDelimiter(_ch) && _ch != '.') ||
LexRules.IsOperator(_ch) || _finish)
{
Retract();
return true;
}
return false;
}
private bool IsDot()
{
if (_tokens.Count != 0)
{
SemanticToken tokenBefore = _tokens.Last();
if (tokenBefore.TokenType == SemanticTokenType.Identifier) return true;
}
return false;
}
private void HandleDelimiterState()
{
Cat();
switch (_ch)
{
case '.':
{
GetChar();
if (_ch == '.')
{
Cat();
_semanticToken = LexemeFactory.MakeToken(DelimiterType.DoubleDots, "..", _line, _chPos);
break;
}
Retract();
if (IsDot())
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Dot, ".", _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Period, ".", _line, _chPos);
}
}
break;
case '\'':
{
// 重置_token，准备收集字符串内容
ResetTokenBuilder();
GetChar(); // 移动到下一个字符，即字符串的第一个字符
while (_ch != '\'' && _ch != '\"')
{
Cat(); // 收集字符
GetChar(); // 移动到下一个字符
if (_ch == '\n' || _finish)
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos,
"The String is not closed.");
}
}
string currentString = GetCurrentTokenString();
if (currentString.Length > 1)
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.String,
currentString, _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(SemanticTokenType.Character,
currentString, _line, _chPos);
}
ResetTokenBuilder();
if (!(_ch == '\'' || _ch == '\"'))
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos,
"The String is not closed.");
}
}
break;
case ',':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Comma, ",", _line, _chPos);
break;
case ':':
char nextChar = PeekNextChar();
if (nextChar == '=')
{
GetChar();
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Assign, ":=", _line, _chPos);
}
else
{
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Colon, ":", _line, _chPos);
}
break;
case ';':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.Semicolon, ";", _line, _chPos);
break;
case '(':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.LeftParenthesis, "(", _line, _chPos);
break;
case ')':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.RightParenthesis, ")", _line, _chPos);
break;
case '[':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.LeftSquareBracket, "[", _line, _chPos);
break;
case ']':
_semanticToken = LexemeFactory.MakeToken(DelimiterType.RightSquareBracket, "]", _line, _chPos);
break;
}
if (_semanticToken is null)
{
throw new InvalidOperationException();
}
_tokens.Add(_semanticToken);
_state = StateType.Start;
}
private void HandleOperatorState()
{
switch (_ch)
{
case '+': // 识别 +
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Plus, "+", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '-': // 识别 -
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Minus, "-", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '*': // 识别 *
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Multiply, "*", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '/': // 识别 /
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Divide, "/", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '=':
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Equal, "=", _line, _chPos);
AddToTokens(_semanticToken);
break;
case '<':
Cat();
GetChar();
if (_ch == '=')
{
// 识别 <=
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.LessEqual, "<=", _line, _chPos);
AddToTokens(_semanticToken);
}
else if (_ch == '>')
{
// 识别 <>
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.NotEqual, ">", _line, _chPos);
AddToTokens(_semanticToken);
}
else
{
// 识别 <
Retract();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Less, "<", _line, _chPos);
AddToTokens(_semanticToken);
}
break;
case '>':
Cat();
GetChar();
if (_ch == '=')
{
// 识别 >=
Cat();
_semanticToken = LexemeFactory.MakeToken(OperatorType.GreaterEqual, ">=", _line, _chPos);
AddToTokens(_semanticToken);
}
else
{
// 识别 >
Retract();
_semanticToken = LexemeFactory.MakeToken(OperatorType.Greater, ">", _line, _chPos);
AddToTokens(_semanticToken);
}
break;
default:
throw new LexemeException(LexemeErrorType.UnknownCharacterOrString, _line, _chPos, "Illegal lexeme.");
}
_state = StateType.Start;
}
private void AddToTokens(SemanticToken semanticToken)
{
_tokens.Add(semanticToken);
}
private void Cat()
{
_tokenBuilder.Append(_ch); // 使用StringBuilder追加字符
}
private string GetCurrentTokenString()
{
return _tokenBuilder.ToString(); // 从StringBuilder获取当前记号的字符串
}
private void ResetTokenBuilder()
{
_tokenBuilder.Clear(); // 清空StringBuilder以复用
}
private char PeekNextChar()
{
// 确认下一个位置是否仍在buffer的范围内
if (reader.TryPeekChar(out char? c))
{
return c.Value;
}
else
{
return char.MinValue;
}
}
private void GetChar()
{
if (_finish)
{
return;
}
_finish = !reader.MoveNext();
if (_finish)
{
_ch = char.MinValue;
return;
}
_ch = reader.Current;
_line = reader.Line;
_chPos = reader.Pos;
}
private void Retract()
{
reader.Retract();
}
}