post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lexical-parser (#15)

Browse Source 
add: 词法分析器剩下数字、标识符的细节处理以及错误处理

Co-authored-by: duqoo <92306417+duqoo@users.noreply.github.com>
Reviewed-on: PostGuard/Canon#15
Co-authored-by: Huaps <1183155719@qq.com>
Co-committed-by: Huaps <1183155719@qq.com>
This commit is contained in:
Huaps 2024-04-04 21:25:11 +08:00 committed by jackfiled
parent ccd1899739
commit c4189fd1b2
13 changed files with 955 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Canon.Core/Enums/ErrorEnums.cs Normal file
View File

@ -0,0 +1,11 @@
namespace Canon.Core.Enums;
public enum LexemeErrorType
{
IllegalNumberFormat,//数字格式不正确
UnknownCharacterOrString,//源代码包含无法识别的字符或字符串
UnclosedStringLiteral,//字符串字面量未闭合
UnclosedComment,//注释未闭合
InvalidEscapeSequence,//无效的转义字符
IllegalOperator,//非法的操作符
}

2
Canon.Core/Enums/SemanticEnums.cs
View File

@ -93,7 +93,7 @@ public enum StateType
Word,
Digit,
Delimiter,
Other
Operator
}
public enum BasicIdType

23
Canon.Core/Exceptions/LexemeException.cs
View File

@ -1,9 +1,13 @@
namespace Canon.Core.Exceptions;
using Enums;
/// <summary>
/// 词法分析中引发的异常
/// </summary>
public class LexemeException : Exception
{
public LexemeErrorType ErrorType { get; }
public uint Line { get; }
public uint CharPosition { get; }
public LexemeException() { }
public LexemeException(string message) : base(message) { }
@ -11,15 +15,20 @@ public class LexemeException : Exception
public LexemeException(string message, Exception innerException) :
base(message, innerException) { }
/// <param name="errorType">错误类型</param>
/// <param name="line">单词的行号</param>
/// <param name="charPosition">单词的列号</param>
/// <param name="message">错误信息</param>
public LexemeException(uint line, uint charPosition, string message) :
base("line:" + line + ", charPosition:" + charPosition + " :" + message) { }
public LexemeException(LexemeErrorType errorType, uint line, uint charPosition, string message) :
base("line:" + line + ", charPosition:" + charPosition + " :" + message)
{
ErrorType = errorType;
Line = line;
CharPosition = charPosition;
}
public LexemeException(uint line, uint charPosition, Exception innerException) :
base("line:" + line + ", charPosition:" + charPosition + " : ", innerException) { }
public LexemeException(uint line, uint charPosition, string message, Exception innerException) :
base("line:" + line + ", charPosition:" + charPosition + " :" + message, innerException) { }
public override string ToString()
{
return $"LexemeException: ErrorType={ErrorType}, Line={Line}, CharPosition={CharPosition}, Message={Message}\n";
}
}

687
Canon.Core/LexicalParser/Lexer.cs
View File

@ -1,5 +1,7 @@
using System.Numerics;
using System.Text;
using Canon.Core.Enums;
using Canon.Core.Exceptions;
namespace Canon.Core.LexicalParser;
@ -17,27 +19,31 @@ public class Lexer(string source)
"Not", "Mod", "And", "Or"
];
private readonly string[] _delimiter = [";", ",", ":", ".", "(", ")", "[", "]","'","\"",".."];
private readonly string[] _delimiter = [";", ",", ":", ".", "(", ")", "[", "]", "'", "\"", ".."];
private readonly string[] _operator = ["=", "<>", "<", "<=", ">", ">=", "+", "-", "*", "/", ":="];
// 状态机
private StateType _state;
private char _ch;
private LinkedList<char> _token = new LinkedList<char>();
// bool save;
// int saved_state;
bool _finish;
private bool eof;
//缓冲区
private readonly char[] _buffer = new char[2048];
// int start_pos;
private int _fwdPos;
// 计数器
private uint _line = 1;
private uint _chPos;
private int _sourcePos;
private readonly Dictionary<SemanticTokenType, int> _tokenCount = new Dictionary<SemanticTokenType, int>
{
{ SemanticTokenType.Keyword, 0 },
@ -57,234 +63,394 @@ public class Lexer(string source)
// 缓冲区
// start_pos = 0;
_fwdPos = 0;
FillLeftBuffer();
// 状态机
_finish = false;
while (!_finish) {
while (!_finish)
{
GetChar();
GetNbc();
if (_finish) break;
_token = new LinkedList<char>();
if (IsLetter()) {
if (IsLetter())
{
_state = StateType.Word;
}
else if (IsDigit()) {
else if(_ch == '.')
{
char next = PeekNextChar();
if (next >= '0' && next <= '9')
{
_state = StateType.Digit;
}
else
{
_state = StateType.Delimiter;
}
}
else if (IsDigit() || _ch == '$')
{
_state = StateType.Digit;
}
else if (IsDelimiter()) {
else if (IsDelimiter())
{
_state = StateType.Delimiter;
}
else if (_ch == '{')
{
GetChar();
while (_ch != '}')
{
GetChar();
if (_ch == '\n')
{
_line++;
_chPos = 0;
}
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos, "The comment is not closed.");
}
}
continue;
}
else
{
_state = StateType.Other;
_state = StateType.Operator;
}
switch (_state)
{
case StateType.Word: {
while (IsDigit() || IsLetter())
case StateType.Word:
while (IsDigit() || IsLetter())
{
Cat();
GetChar();
}
Retract();
if (IsKeyword())
{
KeywordType keywordType =
KeywordSemanticToken.GetKeywordTypeByKeyword(LinkedListToString(_token.First));
MakeToken(keywordType);
}
else
{
MakeToken(SemanticTokenType.Identifier);
}
break;
case StateType.Digit:
DealNumber();
break;
case StateType.Delimiter:
Cat();
switch (_ch)
{
case '.':
{
GetChar();
if (_ch == '.')
{
Cat();
MakeToken(DelimiterType.DoubleDots);
break;
}
Retract();
if (IsDot())
{
MakeToken(DelimiterType.Dot);
}
else
{
MakeToken(DelimiterType.Period);
}
}
break;
case '\'':
case '\"':
{
// 重置_token，准备收集字符串内容
_token = new LinkedList<char>();
GetChar(); // 移动到下一个字符，即字符串的第一个字符
while (_ch != '\'' && _ch != '\"')
{
Cat(); // 收集字符
GetChar(); // 移动到下一个字符
if (_ch == '\n' || _finish)
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos, "The String is not closed.");
}
}
MakeToken(SemanticTokenType.Character); // 或其它适用于字符串字面量的SemanticTokenType
_token = new LinkedList<char>(); // 重置_token
if (!(_ch == '\'' || _ch == '\"'))
{
throw new LexemeException(LexemeErrorType.UnclosedStringLiteral, _line, _chPos, "The String is not closed.");
}
}
break;
case ',':
MakeToken(DelimiterType.Comma);
break;
case ':':
char nextChar = PeekNextChar();
if (nextChar == '=')
{
GetChar();
Cat();
MakeToken(OperatorType.Assign);
}
else
{
MakeToken(DelimiterType.Colon);
}
break;
case ';':
MakeToken(DelimiterType.Semicolon);
break;
case '(':
char next = PeekNextChar();
if (next == '*')
{
GetChar();
bool commentClosed = false;
while (!commentClosed)
{
GetNbc();
GetChar();
while (_ch != '*')
{
GetNbc();
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos, "The comment is not closed.");
}
}
GetChar();
if (_finish)
{
throw new LexemeException(LexemeErrorType.UnclosedComment, _line, _chPos, "The comment is not closed.");
}
if (_ch == ')') commentClosed = true;
}
}
else
{
MakeToken(DelimiterType.LeftParenthesis);
}
break;
case ')':
MakeToken(DelimiterType.RightParenthesis);
break;
case '[':
MakeToken(DelimiterType.LeftSquareBracket);
break;
case ']':
MakeToken(DelimiterType.RightSquareBracket);
break;
}
break;
case StateType.Operator:
DealOther();
break;
default:
throw new ArgumentOutOfRangeException();
}
}
return _tokens;
}
private void DealNumber()
{
// 十六进制
if (_ch == '$')
{
Cat();
GetChar();
while (!NumberShouldBreak())
{
// 假设IsHexDigit方法能够识别十六进制数字
if (IsHexDigit())
{
Cat();
GetChar();
}
Retract();
if (IsKeyword())
else if(NumberShouldBreak())
{
KeywordType keywordType =
KeywordSemanticToken.GetKeywordTypeByKeyword(LinkedListToString(_token.First));
MakeToken(keywordType);
break;
}
else {
MakeToken(SemanticTokenType.Identifier);
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal hex numbers!");
}
break;
}
MakeToken(NumberType.Hex);
return;
}
case StateType.Digit:
// 非十六进制
if(IsDigit() || _ch == '.')
{
while (!NumberShouldBreak())
{
// 含小数部分
if (_ch == '.')
{
bool error = false;
bool tag = false; // 用于标记是否已经处理过科学记数法的指数部分
bool doubleDot = false;
NumberType numberType = NumberType.Integer;
while (IsDigit() || _ch == '.' || _ch == 'E' || _ch == '+' || _ch == '-' || _ch == 'e' || IsLetter()) {
if (_ch != '.')
{
Cat();
}
if (_ch == '0' && !tag) {
GetChar();
if (_ch == 'x' || _ch == 'X') {
numberType = NumberType.Hex; // 标识十六进制
Cat();
while (IsHexDigit()) { // 假设IsHexDigit方法能够识别十六进制数字
Cat();
}
break;
}
Retract(); // 如果不是'x'或'X'，回退一个字符
}
else if (_ch == '.') {
GetChar();
if (_ch == '.') {
Retract(); // 回退到第一个'.'
Retract(); // 回退到'.'之前的数字
doubleDot = true;
break;
}
Retract();
Cat();
numberType = NumberType.Real;
}
else if ((_ch == 'e' || _ch == 'E') && !tag) {
GetChar();
if (IsDigit() || _ch == '+' || _ch == '-') {
Cat();
tag = true; // 已处理指数部分
continue;
}
error = true; // 错误的科学记数法
break;
}
GetChar();
}
if (!error) {
MakeToken(numberType);
if (doubleDot)
{
break;
}
Retract();
}
else
// 检查是否是符号 “..”
char next = PeekNextChar();
if (next == '.')
{
Retract();
PrintError(0,_token.First,_line);
_tokenCount[SemanticTokenType.Error]++;
break;
}
break;
}
case StateType.Delimiter:
Cat();
switch (_ch)
{
case '.':
// 不是符号 “..”,进入小数点后的判断
Cat(); // 记录“.”
// “.”后不应为空，至少应该有一位小数
GetChar();
if (NumberShouldBreak())
{
GetChar();
if (_ch == '.')
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal numbers!");
}
// 读取小数点后的数字
while (!NumberShouldBreak())
{
if (IsDigit())
{
Cat();
MakeToken(DelimiterType.DoubleDots);
GetChar();
}
else if (_ch == 'e' || _ch == 'E')
{
DealE();
break;
}
Retract();
if (IsPeriod())
else if(NumberShouldBreak())
{
}else if (IsDot())
{
}
}
break;
case '\'':
case '\"':
{
if(_ch == '\'') MakeToken(DelimiterType.SingleQuotation);
else if(_ch == '\"') MakeToken(DelimiterType.DoubleQuotation);
// 重置_token，准备收集字符串内容
_token = new LinkedList<char>();
GetChar(); // 移动到下一个字符，即字符串的第一个字符
while (_ch != '\'' && _ch != '\"')
{
Cat(); // 收集字符
GetChar(); // 移动到下一个字符
}
// 在退出循环时，_ch为'或EOF，此时_token包含字符串内容
// 创建字符内容的token，注意这里使用SemanticTokenType.String表示字符串字面量
MakeToken(SemanticTokenType.Character); // 或其它适用于字符串字面量的SemanticTokenType
_token = new LinkedList<char>(); // 重置_token
if (_ch == '\'' && _ch != '\n')
{
// 识别并创建最后一个单引号的token
Cat();
MakeToken(DelimiterType.SingleQuotation);
}
else if (_ch == '\"')
{
Cat();
MakeToken(DelimiterType.DoubleQuotation);
break;
}
else
{
// 这里处理遇到EOF但没有闭合单引号的情况，例如：'字符串结尾没有单引号
// 可以添加错误处理代码
PrintError(0, _token.First, _line); // 假设这个方法用于打印错误
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
break;
case ',':
MakeToken(DelimiterType.Comma);
break;
case ':':
MakeToken(DelimiterType.Colon);
break;
case ';':
MakeToken(DelimiterType.Semicolon);
break;
case '(':
MakeToken(DelimiterType.LeftParenthesis);
break;
case ')':
MakeToken(DelimiterType.RightParenthesis);
break;
case '[':
MakeToken(DelimiterType.LeftSquareBracket);
break;
case ']':
MakeToken(DelimiterType.RightSquareBracket);
MakeToken(NumberType.Real);
return;
}
// 不含小数部分，含科学计数法
if (_ch == 'e' || _ch == 'E')
{
DealE();
MakeToken(NumberType.Real);
return;
}
// 暂时为整数
if (IsDigit())
{
Cat();
GetChar();
}
else if(NumberShouldBreak())
{
break;
}
break;
case StateType.Other:
DealOther();
break;
default:
throw new ArgumentOutOfRangeException();
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
MakeToken(NumberType.Integer);
}
PrintResult();
return _tokens;
}
private void DealE()
{
Cat();
GetChar();
if (IsDigit() || _ch == '+' || _ch == '-')
{
Cat();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
// 读取e后的数字
GetChar();
while (!NumberShouldBreak())
{
if (IsDigit())
{
Cat();
GetChar();
}
else
{
throw new LexemeException(LexemeErrorType.IllegalNumberFormat, _line, _chPos, "Illegal number.");
}
}
}
bool NumberShouldBreak()
{
if (_ch == ' ' || _ch == '\n' || _ch == '\t' || _ch == '\r' || (IsDelimiter() && _ch!='.') || IsOperator() || _finish)
{
Retract();
return true;
}
return false;
}
private bool IsOperator()
{
foreach (var o in _operator)
{
if (o.Contains(_ch))
{
return true;
}
}
return false;
}
private bool IsDot()
{
SemanticToken tokenBefore = _tokens.Last();
if (tokenBefore.TokenType == SemanticTokenType.Identifier) return true;
if (_tokens.Count != 0)
{
SemanticToken tokenBefore = _tokens.Last();
if (tokenBefore.TokenType == SemanticTokenType.Identifier) return true;
}
return false;
}
private bool IsPeriod()
{
SemanticToken tokenBefore = _tokens.Last();
if (tokenBefore.TokenType == SemanticTokenType.Keyword) return true;
return false;
}
private void DealOther()
{
@ -348,28 +514,8 @@ public class Lexer(string source)
MakeToken(OperatorType.Greater);
}
break;
case ':':
Cat();
GetChar();
if (_ch == '=')
{
// 识别 :=
Cat();
MakeToken(OperatorType.Assign);
}
else
{
// 这里应该被识别为delimiter逻辑上
Cat();
PrintError(1, _token.First, _line);
_tokenCount[SemanticTokenType.Error]++;
}
break;
default:
Cat();
PrintError(1, _token.First, _line);
_tokenCount[SemanticTokenType.Error]++;
break;
throw new LexemeException(LexemeErrorType.UnknownCharacterOrString, _line, _chPos, "Illegal lexeme.");
}
}
@ -396,14 +542,6 @@ public class Lexer(string source)
};
token = identifierSemanticToken;
break;
case SemanticTokenType.Error:
ErrorSemanticToken errorSemanticToken = new ErrorSemanticToken()
{
LinePos = _line, CharacterPos = _chPos, LiteralValue = LinkedListToString(_token.First),
};
token = errorSemanticToken;
break;
default:
throw new ArgumentOutOfRangeException(nameof(tokenType), tokenType, null);
}
@ -449,6 +587,32 @@ public class Lexer(string source)
Console.WriteLine(LinkedListToString(_token.First));
}
private void MakeToken(NumberType numberType)
{
string temp = LinkedListToString(_token.First);
string result;
if (numberType == NumberType.Hex)
{
result = string.Concat("0x", temp.AsSpan(1, temp.Length - 1));
}
else
{
result = temp;
}
NumberSemanticToken numberSemanticToken = new NumberSemanticToken()
{
LinePos = _line,
CharacterPos = _chPos,
LiteralValue = result,
NumberType = numberType
};
_tokens.Add(numberSemanticToken);
_tokenCount[SemanticTokenType.Number]++;
Console.WriteLine($"<{SemanticTokenType.Number}> <{numberType}>");
Console.WriteLine(LinkedListToString(_token.First));
}
private void MakeToken(OperatorType operatorType)
{
OperatorSemanticToken operatorSemanticToken = new OperatorSemanticToken()
@ -464,88 +628,20 @@ public class Lexer(string source)
Console.WriteLine(LinkedListToString(_token.First));
}
private void MakeToken(NumberType numberType)
{
NumberSemanticToken numberSemanticToken = new NumberSemanticToken()
{
LinePos = _line,
CharacterPos = _chPos,
LiteralValue = LinkedListToString(_token.First),
NumberType = numberType
};
_tokens.Add(numberSemanticToken);
_tokenCount[SemanticTokenType.Number]++;
Console.WriteLine($"<{SemanticTokenType.Number}> <{numberType}>");
Console.WriteLine(LinkedListToString(_token.First));
}
// 填充buffer操作
private void FillLeftBuffer() {
//cout << "fill left" << endl;
for (int i = 0; i < _buffer.Length / 2; i++) {
_buffer[i] = '$';
}
// 确保source字符串足够长，避免超出范围
int lengthToCopy = Math.Min(_buffer.Length / 2 - 1, source.Length - _sourcePos);
// 使用Array.Copy方法
Array.Copy(source.ToCharArray(), _sourcePos, _buffer, 0, lengthToCopy);
_sourcePos += lengthToCopy;
if (_sourcePos == source.Length) {
eof = true;
}
}
private void FillRightBuffer() {
//cout << "fill right" << endl;
for (int i = _buffer.Length / 2; i < _buffer.Length; i++) {
_buffer[i] = '$';
}
// 确保source字符串足够长，避免超出范围
int lengthToCopy = Math.Min(_buffer.Length / 2 - 1, source.Length - _sourcePos);
// 使用Array.Copy方法
Array.Copy(source.ToCharArray(), _sourcePos, _buffer, _buffer.Length / 2, lengthToCopy);
_sourcePos += lengthToCopy;
if (_sourcePos == source.Length) {
eof = true;
}
}
private void PrintBuffer() {
for (int i = 0; i < _buffer.Length; i++) {
Console.WriteLine($"[{i}] {_buffer[i]}");
}
}
void DealEof() {
if (eof) _finish = true;
else if (_fwdPos < _buffer.Length / 2) {
FillRightBuffer();
_fwdPos = _buffer.Length / 2;
}
else {
FillLeftBuffer();
// start_pos = 0;
_fwdPos = 0;
}
}
// 读取buffer操作
// 读取字符操作
void GetChar() {
if (_fwdPos >= 0 && _fwdPos < _buffer.Length) _ch = _buffer[_fwdPos];
_chPos++;
if (_ch == '$') {
DealEof();
if (_fwdPos >= 0 && _fwdPos < _buffer.Length) _ch = _buffer[_fwdPos];
if (_fwdPos >= 0 && _fwdPos < source.Length)
{
_ch = source[_fwdPos];
_chPos++;
_fwdPos++;
}
else if (_fwdPos == source.Length)
{
_ch = '\0';
_chPos++;
_finish = true;
}
if (_fwdPos < _buffer.Length) _fwdPos++;
}
private void GetNbc() {
@ -622,24 +718,25 @@ public class Lexer(string source)
{
if (delimiter.Contains(_ch))
{
if (_ch != ':')
{
return true;
}
GetChar();
if (_ch == '=')
{
Retract();
return false;
}
return true;
}
}
return false;
}
private char PeekNextChar()
{
// 确认下一个位置是否仍在buffer的范围内
if (_fwdPos < source.Length)
{
return source[_fwdPos];
}
return '\0';
}
private void PrintToken(SemanticTokenType type, LinkedListNode<char> token, uint line)
{
string tokenString = LinkedListToString(token);

134
Canon.Core/LexicalParser/SemanticToken.cs
View File

@ -4,12 +4,10 @@ namespace Canon.Core.LexicalParser;
using Enums;
using System.Text;
/// <summary>
/// 词法记号基类
/// </summary>
public abstract class SemanticToken
public abstract class SemanticToken : IEquatable<SemanticToken>
{
public abstract SemanticTokenType TokenType { get; }
@ -59,7 +57,34 @@ public abstract class SemanticToken
LinePos = 0, CharacterPos = 0, LiteralValue = string.Empty
};
public override string ToString() => LiteralValue;
public override string ToString()
{
return $"LinePos: {LinePos}, CharacterPos: {CharacterPos}, LiteralValue: {LiteralValue}, TokenType: {TokenType}";
}
public bool Equals(SemanticToken? other)
{
if (other == null)
return false;
return LinePos == other.LinePos &&
CharacterPos == other.CharacterPos &&
LiteralValue == other.LiteralValue &&
TokenType == other.TokenType;
}
public override bool Equals(object? obj)
{
return obj is SemanticToken semanticTokenObj && Equals(semanticTokenObj);
}
public override int GetHashCode()
{
return LinePos.GetHashCode() ^
CharacterPos.GetHashCode() ^
LiteralValue.GetHashCode() ^
TokenType.GetHashCode();
}
}
/// <summary>
@ -118,6 +143,11 @@ public class DelimiterSemanticToken : SemanticToken
};
return true;
}
public override int GetHashCode()
{
return base.GetHashCode() ^ this.DelimiterType.GetHashCode();
}
}
/// <summary>
@ -218,6 +248,11 @@ public class KeywordSemanticToken : SemanticToken
token = null;
return false;
}
public override int GetHashCode()
{
return base.GetHashCode() ^ this.KeywordType.GetHashCode();
}
}
/// <summary>
@ -229,12 +264,44 @@ public class OperatorSemanticToken : SemanticToken
public required OperatorType OperatorType { get; init; }
public static readonly Dictionary<string, OperatorType> OperatorTypes = new Dictionary<string, OperatorType>
{
{ "=", OperatorType.Equal },
{ "<>", OperatorType.NotEqual },
{ "<", OperatorType.Less },
{ "<=", OperatorType.LessEqual },
{ ">", OperatorType.Greater },
{ ">=", OperatorType.GreaterEqual },
{ "+", OperatorType.Plus },
{ "-", OperatorType.Minus },
{ "*", OperatorType.Multiply },
{ "/", OperatorType.Divide },
{ ":=", OperatorType.Assign }
};
public static OperatorType GetOperatorTypeByOperator(string operatorSymbol)
{
if (OperatorTypes.TryGetValue(operatorSymbol, out var operatorType))
{
return operatorType;
}
else
{
throw new ArgumentException($"Unknown operator: {operatorSymbol}");
}
}
public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
out OperatorSemanticToken? token)
{
token = null;
return false;
}
public override int GetHashCode()
{
return base.GetHashCode() ^ this.OperatorType.GetHashCode();
}
}
/// <summary>
@ -245,65 +312,10 @@ public class NumberSemanticToken : SemanticToken
public override SemanticTokenType TokenType => SemanticTokenType.Number;
public required NumberType NumberType { get; init; }
public double Value { get; private init; }
public static bool TryParse(uint linePos, uint characterPos, LinkedListNode<char> now,
out NumberSemanticToken? token)
public override int GetHashCode()
{
StringBuilder buffer = new();
bool hasDecimalPoint = false;
bool hasExponent = false;
bool hasMinusSign = false;
while (now != null && (char.IsDigit(now.Value) || now.Value == '.' || now.Value == 'e' || now.Value == 'E' || now.Value == '-' || now.Value == '+'))
{
if (now.Value == '.')
{
if (hasDecimalPoint)
{
break;
}
hasDecimalPoint = true;
}
if (now.Value == 'e' || now.Value == 'E')
{
if (hasExponent)
{
break;
}
hasExponent = true;
}
if (now.Value == '-' || now.Value == '+')
{
if (hasMinusSign)
{
break;
}
hasMinusSign = true;
}
buffer.Append(now.Value);
now = now.Next;
}
if (double.TryParse(buffer.ToString(), out double value))
{
token = new NumberSemanticToken
{
LinePos = linePos,
CharacterPos = characterPos,
LiteralValue = buffer.ToString(),
Value = value,
NumberType = hasDecimalPoint || hasExponent ? NumberType.Real : NumberType.Integer
};
return true;
}
token = null;
return false;
return base.GetHashCode() ^ this.NumberType.GetHashCode();
}
}

54
Canon.Tests/LexicalParserTests/CharacterTypeTests.cs Normal file
View File

@ -0,0 +1,54 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
using Xunit.Abstractions;
using Canon.Core.Exceptions;
namespace Canon.Tests.LexicalParserTests
{
public class CharacterTypeTests
{
private readonly ITestOutputHelper _testOutputHelper;
public CharacterTypeTests(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
}
[Theory]
[InlineData("'a'", "a")]
[InlineData("'Hello, World!'", "Hello, World!")]
public void TestCharacterType(string input, string? expectedResult)
{
Lexer lexer = new(input);
if (expectedResult == null)
{
Assert.Throws<LexemeException>(() => lexer.Tokenize());
}
else
{
List<SemanticToken> tokens = lexer.Tokenize();
_testOutputHelper.WriteLine(tokens[0].LiteralValue);
Assert.Single(tokens);
Assert.Equal(SemanticTokenType.Character, tokens[0].TokenType);
Assert.Equal(expectedResult, tokens[0].LiteralValue);
}
}
[Theory]
//[InlineData("'\\x'", 1, 2, LexemeException.LexemeErrorType.InvalidEscapeSequence)]
[InlineData("'This is an unclosed string literal", 1, 36, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("'This", 1, 6, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("x @", 1, 3, LexemeErrorType.UnknownCharacterOrString)]
//[InlineData("\"x\'", 1, 3, LexemeException.LexemeErrorType.UnclosedStringLiteral)]
public void TestParseCharacterError(string input, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)
{
Lexer lexer = new(input);
var ex = Assert.Throws<LexemeException>(() => lexer.Tokenize());
_testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(expectedErrorType, ex.ErrorType);
Assert.Equal(expectedLine, ex.Line);
Assert.Equal(expectedCharPosition, ex.CharPosition);
}
}
}

2
Canon.Tests/LexicalParserTests/DelimiterTests.cs
View File

@ -7,7 +7,7 @@ public class DelimiterTests
{
[Theory]
[InlineData(",123", DelimiterType.Comma)]
[InlineData(".123", DelimiterType.Period)]
// [InlineData(".123", DelimiterType.Period)]
[InlineData(":123", DelimiterType.Colon)]
[InlineData(";123", DelimiterType.Semicolon)]
[InlineData("(123)", DelimiterType.LeftParenthesis)]

32
Canon.Tests/LexicalParserTests/ErrorSingleTests.cs Normal file
View File

@ -0,0 +1,32 @@
using Canon.Core.LexicalParser;
using Canon.Core.Exceptions;
using Xunit.Abstractions;
using Canon.Core.Enums;
namespace Canon.Tests.LexicalParserTests
{
public class ErrorSingleTests
{
private readonly ITestOutputHelper _testOutputHelper;
public ErrorSingleTests(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
}
[Theory]
[InlineData("program main; var a: integer; begin a := 3#; end.", 1, 43, LexemeErrorType.IllegalNumberFormat)]
[InlineData("char c = 'abc;", 1, 15, LexemeErrorType.UnclosedStringLiteral)]
[InlineData("x := 10 @;", 1, 9, LexemeErrorType.UnknownCharacterOrString)]
[InlineData("identifier_with_special_chars@#",1, 30, LexemeErrorType.UnknownCharacterOrString)]
public void TestUnknownCharacterError(string pascalProgram, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)
{
var lexer = new Lexer(pascalProgram);
var ex = Assert.Throws<LexemeException>(() => lexer.Tokenize());
_testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(expectedErrorType, ex.ErrorType);
Assert.Equal(expectedLine, ex.Line);
Assert.Equal(expectedCharPosition, ex.CharPosition);
}
}
}

10
Canon.Tests/LexicalParserTests/IndentifierTests.cs → Canon.Tests/LexicalParserTests/IndentifierTypeTests.cs
View File

@ -1,6 +1,5 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
using Xunit;
namespace Canon.Tests.LexicalParserTests
{
@ -10,20 +9,15 @@ namespace Canon.Tests.LexicalParserTests
[InlineData("identifier", true)]
[InlineData("_identifier", true)]
[InlineData("identifier123", true)]
[InlineData("123identifier", false)]
[InlineData("identifier_with_underscores", true)]
[InlineData("IdentifierWithCamelCase", true)]
[InlineData("identifier-with-hyphen", false)]
[InlineData("identifier with spaces", false)]
[InlineData("identifier_with_special_chars@#", false)]
[InlineData("", false)]
[InlineData(" ", false)]
[InlineData("andand", false)]
[InlineData("andand", true)]
public void TestParseIdentifier(string input, bool expectedResult)
{
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
Assert.Single(tokens);
Assert.Equal(expectedResult, tokens.FirstOrDefault()?.TokenType == SemanticTokenType.Identifier);
}
}

1
Canon.Tests/LexicalParserTests/KeywordTypeTests.cs
View File

@ -21,6 +21,7 @@ public class KeywordTypeTests
[InlineData("for", KeywordType.For)]
[InlineData("to", KeywordType.To)]
[InlineData("do", KeywordType.Do)]
[InlineData("DO", KeywordType.Do)]
public void SmokeTest(string input, KeywordType type)
{
Lexer lexer = new(input);

312
Canon.Tests/LexicalParserTests/LexicalFileTests.cs Normal file
View File

@ -0,0 +1,312 @@
using System.Text.RegularExpressions;
using Canon.Core.Enums;
using Canon.Core.Exceptions;
using Canon.Core.LexicalParser;
using Xunit.Abstractions;
namespace Canon.Tests.LexicalParserTests;
public class LexicalFileTests
{
private readonly ITestOutputHelper _testOutputHelper;
public LexicalFileTests(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
}
//TODO: 基础的字符串匹配，因此变量名称不能被包含。手写一个存在包含情况的测试文件。
private static (int, int) FindNthPosition(string pascalProgram, string target, int occurrence)
{
int lineNumber = 0;
(int, int) nthPosition = (0, 0);
int foundCount = 0;
occurrence = occurrence + 1;
using (StringReader sr = new StringReader(pascalProgram))
{
string line;
while ((line = sr.ReadLine()) != null)
{
lineNumber++;
int columnNumber = -1;
// line = Regex.Replace(line, "'[^']*'", "$");
while ((columnNumber = line.IndexOf(target, columnNumber + 1, StringComparison.Ordinal)) != -1)
{
foundCount++;
if (foundCount == occurrence)
{
nthPosition = (lineNumber, columnNumber + target.Length);
return nthPosition;
}
}
}
}
if (nthPosition == (0, 0))
{
throw new Exception($"'{target}' not found in program.");
}
return nthPosition;
}
private void TestLexicalAnalysis(string pascalProgram, List<(string, SemanticTokenType, int)> stringLiterals)
{
var expectedTokens = new List<SemanticToken>();
foreach (var (literal, tokenType, skipCount) in stringLiterals)
{
var (line, column) = FindNthPosition(pascalProgram, literal, skipCount);
switch (tokenType)
{
case SemanticTokenType.Keyword:
expectedTokens.Add(new KeywordSemanticToken
{
LinePos = (uint)line,
CharacterPos = (uint)column,
LiteralValue = literal,
KeywordType = KeywordSemanticToken.GetKeywordTypeByKeyword(literal)
});
break;
case SemanticTokenType.Identifier:
expectedTokens.Add(new IdentifierSemanticToken
{
LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal
});
break;
case SemanticTokenType.Delimiter:
if (DelimiterSemanticToken.TryParse((uint)line, (uint)column, new LinkedListNode<char>(literal[0]),
out var delimiterToken))
{
if (delimiterToken != null)
{
expectedTokens.Add(delimiterToken);
}
}
break;
case SemanticTokenType.Operator:
expectedTokens.Add(new OperatorSemanticToken
{
LinePos = (uint)line,
CharacterPos = (uint)column,
LiteralValue = literal,
OperatorType = OperatorSemanticToken.GetOperatorTypeByOperator(literal)
});
break;
case SemanticTokenType.Character:
expectedTokens.Add(new CharacterSemanticToken
{
LinePos = (uint)line, CharacterPos = (uint)column, LiteralValue = literal
});
break;
case SemanticTokenType.Number:
expectedTokens.Add(new NumberSemanticToken
{
LinePos = (uint)line,
CharacterPos = (uint)column,
LiteralValue = literal,
NumberType = NumberType.Integer
});
break;
}
}
expectedTokens = expectedTokens.OrderBy(token => token.LinePos).ThenBy(token => token.CharacterPos).ToList();
expectedTokens = expectedTokens.Select(token =>
token is CharacterSemanticToken characterToken && characterToken.LiteralValue == "hello, world!"
? new CharacterSemanticToken
{
LinePos = characterToken.LinePos,
CharacterPos = characterToken.CharacterPos + 1,
LiteralValue = characterToken.LiteralValue
}
: token).ToList();
var lexer = new Lexer(pascalProgram);
var actualTokens = lexer.Tokenize();
for (int i = 0; i < expectedTokens.Count; i++)
{
_testOutputHelper.WriteLine($"Expect: {expectedTokens[i]}");
_testOutputHelper.WriteLine($"Actual: {actualTokens[i]}");
_testOutputHelper.WriteLine("----");
Assert.Equal(expectedTokens[i], actualTokens[i]);
}
Assert.Equal(expectedTokens, actualTokens);
}
[Fact]
public void TestLexicalAnalysisFirst()
{
string pascalProgram = """
program HelloWorld;
var
message: string;
begin
message := 'hello, world!';
writeln(message);
end.
""";
var stringLiterals = new List<(string, SemanticTokenType, int)>
{
("program", SemanticTokenType.Keyword, 0),
("HelloWorld", SemanticTokenType.Identifier, 0),
(";", SemanticTokenType.Delimiter, 0),
("var", SemanticTokenType.Keyword, 0),
("message", SemanticTokenType.Identifier, 0),
(":", SemanticTokenType.Delimiter, 0),
("string", SemanticTokenType.Identifier, 0),
(";", SemanticTokenType.Delimiter, 1),
("begin", SemanticTokenType.Keyword, 0),
("message", SemanticTokenType.Identifier, 1),
(":=", SemanticTokenType.Operator, 0),
("hello, world!", SemanticTokenType.Character, 0),
(";", SemanticTokenType.Delimiter, 2),
("writeln", SemanticTokenType.Identifier, 0),
("(", SemanticTokenType.Delimiter, 0),
("message", SemanticTokenType.Identifier, 2),
(")", SemanticTokenType.Delimiter, 0),
(";", SemanticTokenType.Delimiter, 3),
("end", SemanticTokenType.Keyword, 0),
(".", SemanticTokenType.Delimiter, 0)
};
TestLexicalAnalysis(pascalProgram, stringLiterals);
}
[Fact]
public void TestLexicalAnalysisSecond()
{
string pascalProgram = """
program main;
var
ab: integer;
begin
ab := 3;
write(ab);
end.
""";
var stringLiterals = new List<(string, SemanticTokenType, int)>
{
("program", SemanticTokenType.Keyword, 0),
("main", SemanticTokenType.Identifier, 0),
(";", SemanticTokenType.Delimiter, 0),
("var", SemanticTokenType.Keyword, 0),
("ab", SemanticTokenType.Identifier, 0),
(":", SemanticTokenType.Delimiter, 0),
("integer", SemanticTokenType.Keyword, 0),
(";", SemanticTokenType.Delimiter, 1),
("begin", SemanticTokenType.Keyword, 0),
("ab", SemanticTokenType.Identifier, 1),
(":=", SemanticTokenType.Operator, 0),
("3", SemanticTokenType.Number, 0),
(";", SemanticTokenType.Delimiter, 2),
("write", SemanticTokenType.Identifier, 0),
("(", SemanticTokenType.Delimiter, 0),
("ab", SemanticTokenType.Identifier, 2),
(")", SemanticTokenType.Delimiter, 0),
(";", SemanticTokenType.Delimiter, 3),
("end", SemanticTokenType.Keyword, 0),
(".", SemanticTokenType.Delimiter, 0)
};
TestLexicalAnalysis(pascalProgram, stringLiterals);
}
//带注释的测试
[Fact]
public void TestLexicalAnalysisThird()
{
string pascalProgram = """
{test}
program main;
var
ab, ba: integer;
begin
ab := 3;
ba := 5;
ab := 5;
write(ab + ba);
end.
""";
var stringLiterals = new List<(string, SemanticTokenType, int)>
{
("program", SemanticTokenType.Keyword, 0),
("main", SemanticTokenType.Identifier, 0),
(";", SemanticTokenType.Delimiter, 0),
("var", SemanticTokenType.Keyword, 0),
("ab", SemanticTokenType.Identifier, 0),
(",", SemanticTokenType.Delimiter, 0),
("ba", SemanticTokenType.Identifier, 0),
(":", SemanticTokenType.Delimiter, 0),
("integer", SemanticTokenType.Keyword, 0),
(";", SemanticTokenType.Delimiter, 1),
("begin", SemanticTokenType.Keyword, 0),
("ab", SemanticTokenType.Identifier, 1),
(":=", SemanticTokenType.Operator, 0),
("3", SemanticTokenType.Number, 0),
(";", SemanticTokenType.Delimiter, 2),
("ba", SemanticTokenType.Identifier, 1),
(":=", SemanticTokenType.Operator, 1),
("5", SemanticTokenType.Number, 0),
(";", SemanticTokenType.Delimiter, 3),
("ab", SemanticTokenType.Identifier, 2),
(":=", SemanticTokenType.Operator, 2),
("5", SemanticTokenType.Number, 1),
(";", SemanticTokenType.Delimiter, 4),
("write", SemanticTokenType.Identifier, 0),
("(", SemanticTokenType.Delimiter, 0),
("ab", SemanticTokenType.Identifier, 3),
("+", SemanticTokenType.Operator, 0),
("ba", SemanticTokenType.Identifier, 2),
(")", SemanticTokenType.Delimiter, 0),
(";", SemanticTokenType.Delimiter, 5),
("end", SemanticTokenType.Keyword, 0),
(".", SemanticTokenType.Delimiter, 0)
};
TestLexicalAnalysis(pascalProgram, stringLiterals);
}
[Fact]
public void UnclosedCommentFirst()
{
string pascalProgram = """
(* This is an example of an unclosed comment
program CommentError;
var
x: integer;
begin
x := 42;
end.
""";
var lexer = new Lexer(pascalProgram);
var ex = Assert.Throws<LexemeException>(() => lexer.Tokenize());
//打印exception信息
_testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
Assert.Equal((uint)7, ex.Line);
Assert.Equal((uint)5, ex.CharPosition);
}
[Fact]
public void UnclosedCommentSecond()
{
string pascalProgram = """
{
This is a block comment that does not close.
program CommentNotClosed;
""";
var lexer = new Lexer(pascalProgram);
var ex = Assert.Throws<LexemeException>(() => lexer.Tokenize());
_testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(LexemeErrorType.UnclosedComment, ex.ErrorType);
Assert.Equal((uint)4, ex.Line);
Assert.Equal((uint)26, ex.CharPosition);
}
}

68
Canon.Tests/LexicalParserTests/NumberTests.cs
View File

@ -1,46 +1,58 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
using Canon.Core.Exceptions;
using Xunit.Abstractions;
namespace Canon.Tests.LexicalParserTests
{
public class NumberTests
{
private readonly ITestOutputHelper _testOutputHelper;
public NumberTests(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
}
[Theory]
[InlineData("123", 123, NumberType.Integer)]
[InlineData("0", 0, NumberType.Integer)]
[InlineData("-123", -123, NumberType.Integer)]
[InlineData("1.23", 1.23, NumberType.Real)]
[InlineData("-1.23", -1.23, NumberType.Real)]
[InlineData("0.0", 0.0, NumberType.Real)]
[InlineData("1e7", 1e7, NumberType.Real)]
[InlineData("1E7", 1E7, NumberType.Real)]
[InlineData("1.23e-7", 1.23e-7, NumberType.Real)]
[InlineData("1.23E-7", 1.23E-7, NumberType.Real)]
[InlineData("1234567890", 1234567890, NumberType.Integer)]
[InlineData("1234567890.1234567890", 1234567890.1234567890, NumberType.Real)]
[InlineData("-1234567890", -1234567890, NumberType.Integer)]
[InlineData("-1234567890.1234567890", -1234567890.1234567890, NumberType.Real)]
[InlineData("1e-7", 1e-7, NumberType.Real)]
[InlineData("1E-7", 1E-7, NumberType.Real)]
[InlineData("1E", 0, NumberType.Real, false)]
[InlineData("abc", 0, NumberType.Integer, false)]
[InlineData("123abc", 123, NumberType.Integer, true)]
public void TestParseNumber(string input, double expected, NumberType expectedNumberType,
bool expectedResult = true)
[InlineData("123", "123", NumberType.Integer)]
[InlineData("0", "0", NumberType.Integer)]
[InlineData("1.23", "1.23", NumberType.Real)]
[InlineData("0.0", "0.0", NumberType.Real)]
[InlineData("1e7", "1e7", NumberType.Real)]
[InlineData("1E7", "1E7", NumberType.Real)]
[InlineData("1.23e-7", "1.23e-7", NumberType.Real)]
[InlineData("1.23E-7", "1.23E-7", NumberType.Real)]
[InlineData("1234567890", "1234567890", NumberType.Integer)]
[InlineData("1234567890.1234567890", "1234567890.1234567890", NumberType.Real)]
[InlineData("1e-7", "1e-7", NumberType.Real)]
[InlineData("1E-7", "1E-7", NumberType.Real)]
[InlineData(".67",".67", NumberType.Real)]
[InlineData("$123", "0x123", NumberType.Hex)]
public void TestParseNumber(string input, string expected, NumberType expectedNumberType)
{
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
if (!expectedResult)
{
Assert.NotEqual(SemanticTokenType.Keyword, token.TokenType);
return;
}
Assert.Equal(SemanticTokenType.Number, token.TokenType);
NumberSemanticToken numberSemanticToken = (NumberSemanticToken)token;
Assert.Equal(expectedNumberType, numberSemanticToken.NumberType);
Assert.Equal(expected, numberSemanticToken.Value);
Assert.Equal(expected, numberSemanticToken.LiteralValue);
}
[Theory]
[InlineData("1E", 1, 3, LexemeErrorType.IllegalNumberFormat)]
[InlineData("123abc", 1, 4, LexemeErrorType.IllegalNumberFormat)]
[InlineData("123.45.67", 1, 7, LexemeErrorType.IllegalNumberFormat)]
[InlineData("123identifier", 1, 4, LexemeErrorType.IllegalNumberFormat)]
public void TestParseNumberError(string input, uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)
{
Lexer lexer = new(input);
var ex = Assert.Throws<LexemeException>(() => lexer.Tokenize());
_testOutputHelper.WriteLine(ex.ToString());
Assert.Equal(expectedErrorType, ex.ErrorType);
Assert.Equal(expectedLine, ex.Line);
Assert.Equal(expectedCharPosition, ex.CharPosition);
}
}
}

45
Canon.Tests/LexicalParserTests/OperatorTypeTests.cs
View File

@ -6,38 +6,33 @@ namespace Canon.Tests.LexicalParserTests;
public class OperatorTypeTests
{
[Theory]
[InlineData("+ 123", OperatorType.Plus)]
[InlineData("+123", OperatorType.Plus)]
[InlineData("-123", OperatorType.Minus)]
[InlineData("*123", OperatorType.Multiply)]
[InlineData("/123", OperatorType.Divide)]
[InlineData("=123", OperatorType.Equal)]
[InlineData("<123", OperatorType.Less)]
[InlineData(">123", OperatorType.Greater)]
[InlineData("<=123", OperatorType.LessEqual)]
[InlineData(">=123", OperatorType.GreaterEqual)]
[InlineData("<>123", OperatorType.NotEqual)]
[InlineData(":=123", OperatorType.Assign)]
public void ParseTest(string input, OperatorType result)
[InlineData("+ 123", OperatorType.Plus, true)]
[InlineData("+123", OperatorType.Plus, true)]
[InlineData("-123", OperatorType.Minus, true)]
[InlineData("*123", OperatorType.Multiply, true)]
[InlineData("/123", OperatorType.Divide, true)]
[InlineData("=123", OperatorType.Equal, true)]
[InlineData("<123", OperatorType.Less, true)]
[InlineData(">123", OperatorType.Greater, true)]
[InlineData("<=123", OperatorType.LessEqual, true)]
[InlineData(">=123", OperatorType.GreaterEqual, true)]
[InlineData("<>123", OperatorType.NotEqual, true)]
[InlineData(":=123", OperatorType.Assign, true)]
[InlineData("1 + 123", OperatorType.Plus, false)]
[InlineData("m +123", OperatorType.Plus, false)]
public void ParseTest(string input, OperatorType result, bool expectedResult)
{
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
if (!expectedResult)
{
Assert.NotEqual(SemanticTokenType.Operator, token.TokenType);
return;
}
Assert.Equal(SemanticTokenType.Operator, token.TokenType);
OperatorSemanticToken operatorSemanticToken = (OperatorSemanticToken)token;
Assert.Equal(result, operatorSemanticToken.OperatorType);
}
[Theory]
[InlineData("1 + 123")]
[InlineData("m +123")]
public void ParseFailedTest(string input)
{
Lexer lexer = new(input);
List<SemanticToken> tokens = lexer.Tokenize();
SemanticToken token = tokens[0];
Assert.NotEqual(SemanticTokenType.Operator, token.TokenType);
}
}