refact: syntax-node (#23)

重构语法树的部分,使用单独的类来抽象不同的非终结符节点。
**同时**,将`Pascal`语法的定义从测试项目中移动到核心项目中,在项目中只维护一份对于`Pascal`语法的定义。

Reviewed-on: PostGuard/Canon#23
This commit is contained in:
jackfiled 2024-04-07 16:47:28 +08:00
parent c0a8e25d45
commit 5e3ea6303e
48 changed files with 1273 additions and 708 deletions

View File

@ -1,5 +1,7 @@
using Canon.Core.GrammarParser; using Canon.Core.Enums;
using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser; using Canon.Core.LexicalParser;
using Canon.Core.SyntaxNodes;
namespace Canon.Core.Abstractions; namespace Canon.Core.Abstractions;
@ -12,10 +14,10 @@ public abstract class GrammarParserBase
public abstract NonTerminator Begin { get; } public abstract NonTerminator Begin { get; }
public SyntaxNode Analyse(IEnumerable<SemanticToken> tokens) public SyntaxNodeBase Analyse(IEnumerable<SemanticToken> tokens)
{ {
Stack<AnalyseState> stack = []; Stack<AnalyseState> stack = [];
stack.Push(new AnalyseState(BeginTransformer, new SyntaxNode(SemanticToken.End))); stack.Push(new AnalyseState(BeginTransformer, SyntaxNodeBase.Create(SemanticToken.End)));
using IEnumerator<SemanticToken> enumerator = tokens.GetEnumerator(); using IEnumerator<SemanticToken> enumerator = tokens.GetEnumerator();
if (!enumerator.MoveNext()) if (!enumerator.MoveNext())
@ -37,21 +39,24 @@ public abstract class GrammarParserBase
return top.Node; return top.Node;
} }
SyntaxNode newNode = new(information.Left.Type); List<SyntaxNodeBase> children = [];
NonTerminatorType leftType = information.Left.Type;
for (int i = 0; i < information.Length; i++) for (int i = 0; i < information.Length; i++)
{ {
newNode.Children.Add(stack.Pop().Node); children.Add(stack.Pop().Node);
} }
// 为了符合生成式的顺序而倒序
children.Reverse();
stack.Push(new AnalyseState(stack.Peek().State.ShiftTable[information.Left], stack.Push(new AnalyseState(stack.Peek().State.ShiftTable[information.Left],
newNode)); SyntaxNodeBase.Create(leftType, children)));
continue; continue;
} }
// 如果没有成功归约就进行移进 // 如果没有成功归约就进行移进
if (top.State.ShiftTable.TryGetValue(enumerator.Current, out ITransformer? next)) if (top.State.ShiftTable.TryGetValue(enumerator.Current, out ITransformer? next))
{ {
stack.Push(new AnalyseState(next, new SyntaxNode(enumerator.Current))); stack.Push(new AnalyseState(next, SyntaxNodeBase.Create(enumerator.Current)));
if (enumerator.MoveNext()) if (enumerator.MoveNext())
{ {
continue; continue;
@ -66,5 +71,5 @@ public abstract class GrammarParserBase
} }
} }
private record AnalyseState(ITransformer State, SyntaxNode Node); private record AnalyseState(ITransformer State, SyntaxNodeBase Node);
} }

View File

@ -1,5 +1,4 @@
using Canon.Core.Abstractions; using Canon.Core.Abstractions;
using Canon.Core.LexicalParser;
namespace Canon.Core.GrammarParser; namespace Canon.Core.GrammarParser;
@ -86,6 +85,4 @@ public class Grammar
public IDictionary<Terminator, ReduceInformation> ReduceTable { get; } public IDictionary<Terminator, ReduceInformation> ReduceTable { get; }
= new Dictionary<Terminator, ReduceInformation>(); = new Dictionary<Terminator, ReduceInformation>();
} }
private record AnalyseState(LrState State, SyntaxNode Node);
} }

View File

@ -1,11 +1,10 @@
using Canon.Core.Enums; using Canon.Core.Enums;
using Canon.Core.GrammarParser;
namespace Canon.Tests.GrammarParserTests; namespace Canon.Core.GrammarParser;
public partial class PascalGrammarTests public static class PascalGrammar
{ {
private static readonly Dictionary<NonTerminator, List<List<TerminatorBase>>> s_pascalGrammar = new() public static readonly Dictionary<NonTerminator, List<List<TerminatorBase>>> Grammar = new()
{ {
{ {
// ProgramStart -> ProgramStruct // ProgramStart -> ProgramStruct

View File

@ -1,130 +0,0 @@
using System.Collections;
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.GrammarParser;
/// <summary>
/// 抽象语法树上的节点
/// </summary>
public class SyntaxNode : IEquatable<SyntaxNode>, IEnumerable<SyntaxNode>
{
private readonly SemanticToken? _semanticToken;
private readonly NonTerminatorType _nonTerminatorType;
public bool IsTerminated { get; }
public List<SyntaxNode> Children { get; } = [];
public SyntaxNode(SemanticToken token)
{
IsTerminated = true;
_semanticToken = token;
}
public SyntaxNode(NonTerminatorType nonTerminatorType)
{
IsTerminated = false;
_nonTerminatorType = nonTerminatorType;
}
/// <summary>
/// 获得终结节点包含的记号对象
/// </summary>
/// <returns>词法分析得到的记号对象</returns>
/// <exception cref="InvalidOperationException">在非终结节点上调用该方法</exception>
public SemanticToken GetSemanticToken()
{
if (!IsTerminated)
{
throw new InvalidOperationException("Can not get semantic token from a not terminated node");
}
return _semanticToken!;
}
/// <summary>
/// 获得非终结节点的类型
/// </summary>
/// <returns>非终结节点类型</returns>
/// <exception cref="InvalidOperationException">在终结节点上调用该方法</exception>
public NonTerminatorType GetNonTerminatorType()
{
if (IsTerminated)
{
throw new InvalidOperationException("Can not get non terminated type from a terminated node");
}
return _nonTerminatorType;
}
public IEnumerator<SyntaxNode> GetEnumerator()
{
yield return this;
foreach (SyntaxNode child in Children)
{
foreach (SyntaxNode node in child)
{
yield return node;
}
}
}
public bool Equals(SyntaxNode? other)
{
if (other is null)
{
return false;
}
if (IsTerminated != other.IsTerminated)
{
return false;
}
if (IsTerminated)
{
return GetSemanticToken() == other.GetSemanticToken();
}
else
{
// TODO: 在判等时是否需要判断子节点也相等
return GetNonTerminatorType() == other.GetNonTerminatorType();
}
}
public override bool Equals(object? obj)
{
if (obj is not SyntaxNode other)
{
return false;
}
return Equals(other);
}
public override int GetHashCode()
{
if (IsTerminated)
{
return GetSemanticToken().GetHashCode();
}
else
{
return GetNonTerminatorType().GetHashCode();
}
}
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
public static bool operator ==(SyntaxNode a, SyntaxNode b)
{
return a.Equals(b);
}
public static bool operator !=(SyntaxNode a, SyntaxNode b)
{
return !a.Equals(b);
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class AddOperator : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.AddOperator;
public static AddOperator Create(List<SyntaxNodeBase> children)
{
return new AddOperator { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class BasicType : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.BasicType;
public static BasicType Create(List<SyntaxNodeBase> children)
{
return new BasicType { Children = children };
}
}

View File

@ -0,0 +1,15 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class CompoundStatement : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.CompoundStatement;
public IEnumerable<Statement> Statements => Children[1].Convert<StatementList>().Statements;
public static CompoundStatement Create(List<SyntaxNodeBase> children)
{
return new CompoundStatement { Children = children };
}
}

View File

@ -0,0 +1,55 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class ConstDeclaration : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ConstDeclaration;
/// <summary>
/// 是否递归的声明下一个ConstDeclaration
/// </summary>
public bool IsRecursive { get; private init; }
/// <summary>
/// 获得声明的常量
/// </summary>
public (IdentifierSemanticToken, ConstValue) ConstValue => GetConstValue();
public static ConstDeclaration Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 3)
{
isRecursive = false;
}
else if (children.Count == 5)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new ConstDeclaration { Children = children, IsRecursive = isRecursive };
}
private static IdentifierSemanticToken ConvertToIdentifierSemanticToken(SyntaxNodeBase node)
{
return (IdentifierSemanticToken)node.Convert<TerminatedSyntaxNode>().Token;
}
private (IdentifierSemanticToken, ConstValue) GetConstValue()
{
if (IsRecursive)
{
return (ConvertToIdentifierSemanticToken(Children[2]), Children[4].Convert<ConstValue>());
}
else
{
return (ConvertToIdentifierSemanticToken(Children[0]), Children[2].Convert<ConstValue>());
}
}
}

View File

@ -0,0 +1,43 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class ConstDeclarations : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ConstDeclarations;
/// <summary>
/// 声明的常量列表
/// </summary>
public IEnumerable<(IdentifierSemanticToken, ConstValue)> ConstValues => GetConstValues();
public static ConstDeclarations Create(List<SyntaxNodeBase> children)
{
return new ConstDeclarations { Children = children };
}
private IEnumerable<(IdentifierSemanticToken, ConstValue)> GetConstValues()
{
if (Children.Count == 0)
{
yield break;
}
ConstDeclaration declaration = Children[1].Convert<ConstDeclaration>();
while (true)
{
yield return declaration.ConstValue;
if (declaration.IsRecursive)
{
declaration = declaration.Children[0].Convert<ConstDeclaration>();
}
else
{
break;
}
}
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ConstValue : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ConstValue;
public static ConstValue Create(List<SyntaxNodeBase> children)
{
return new ConstValue { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ElsePart : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ElsePart;
public static ElsePart Create(List<SyntaxNodeBase> children)
{
return new ElsePart { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Expression : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Expression;
public static Expression Create(List<SyntaxNodeBase> children)
{
return new Expression { Children = children };
}
}

View File

@ -0,0 +1,54 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ExpressionList : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ExpressionList;
public bool IsRecursive { get; private init; }
/// <summary>
/// 声明的表达式列表
/// </summary>
public IEnumerable<Expression> Expressions => GetExpressions();
public static ExpressionList Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 1)
{
isRecursive = false;
}
else if (children.Count == 3)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new ExpressionList { Children = children, IsRecursive = isRecursive };
}
private IEnumerable<Expression> GetExpressions()
{
ExpressionList list = this;
while (true)
{
if (list.IsRecursive)
{
yield return list.Children[2].Convert<Expression>();
list = list.Children[0].Convert<ExpressionList>();
}
else
{
yield return list.Children[0].Convert<Expression>();
break;
}
}
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Factor : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Factor;
public static Factor Create(List<SyntaxNodeBase> children)
{
return new Factor { Children = children };
}
}

View File

@ -0,0 +1,31 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class FormalParameter : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.FormalParameter;
/// <summary>
/// 声明的参数列表
/// </summary>
public IEnumerable<Parameter> Parameters => GetParameters();
public static FormalParameter Create(List<SyntaxNodeBase> children)
{
return new FormalParameter { Children = children };
}
private IEnumerable<Parameter> GetParameters()
{
if (Children.Count == 0)
{
yield break;
}
foreach (Parameter parameter in Children[1].Convert<ParameterList>().Parameters)
{
yield return parameter;
}
}
}

View File

@ -0,0 +1,58 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class IdentifierList : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.IdentifierList;
/// <summary>
/// 是否含有递归定义
/// </summary>
public bool IsRecursive { get; private init; }
/// <summary>
/// 声明的标识符列表
/// </summary>
public IEnumerable<IdentifierSemanticToken> Identifiers => GetIdentifiers();
public static IdentifierList Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 1)
{
isRecursive = false;
}
else if (children.Count == 3)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new IdentifierList { IsRecursive = isRecursive, Children = children };
}
private IEnumerable<IdentifierSemanticToken> GetIdentifiers()
{
IdentifierList identifier = this;
while (true)
{
if (identifier.IsRecursive)
{
yield return (IdentifierSemanticToken)identifier.Children[2].Convert<TerminatedSyntaxNode>().Token;
identifier = identifier.Children[0].Convert<IdentifierList>();
}
else
{
yield return (IdentifierSemanticToken)identifier.Children[0].Convert<TerminatedSyntaxNode>().Token;
break;
}
}
}
}

View File

@ -0,0 +1,51 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class IdentifierVarPart : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.IdVarPart;
/// <summary>
/// 是否声明了索引部分
/// </summary>
public bool Exist { get; private init; }
/// <summary>
/// 索引中的位置声明
/// </summary>
public IEnumerable<Expression> Positions => GetPositions();
private IEnumerable<Expression> GetPositions()
{
if (!Exist)
{
yield break;
}
foreach (Expression expression in Children[1].Convert<ExpressionList>().Expressions)
{
yield return expression;
}
}
public static IdentifierVarPart Create(List<SyntaxNodeBase> children)
{
bool exist;
if (children.Count == 0)
{
exist = false;
}
else if (children.Count == 3)
{
exist = true;
}
else
{
throw new InvalidOperationException();
}
return new IdentifierVarPart { Children = children, Exist = exist };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class MultiplyOperator : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.MultiplyOperator;
public static MultiplyOperator Create(List<SyntaxNodeBase> children)
{
return new MultiplyOperator { Children = children };
}
}

View File

@ -0,0 +1,35 @@
using System.Collections;
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public abstract class NonTerminatedSyntaxNode : SyntaxNodeBase, IEnumerable<SyntaxNodeBase>
{
public override bool IsTerminated => false;
public abstract NonTerminatorType Type { get; }
public required List<SyntaxNodeBase> Children { get; init; }
public IEnumerator<SyntaxNodeBase> GetEnumerator()
{
yield return this;
foreach (SyntaxNodeBase child in Children)
{
if (child.IsTerminated)
{
yield return child;
}
NonTerminatedSyntaxNode nonTerminatedNode = child.Convert<NonTerminatedSyntaxNode>();
foreach (SyntaxNodeBase node in nonTerminatedNode)
{
yield return node;
}
}
}
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}

View File

@ -0,0 +1,40 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Parameter : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Parameter;
/// <summary>
/// 是否为引用变量
/// </summary>
public bool IsVar { get; private init; }
/// <summary>
/// 声明的变量名称
/// </summary>
public ValueParameter ValueParameter =>
IsVar ? Children[0].Convert<VarParameter>().ValueParameter : Children[0].Convert<ValueParameter>();
public static Parameter Create(List<SyntaxNodeBase> children)
{
NonTerminatedSyntaxNode node = children[0].Convert<NonTerminatedSyntaxNode>();
bool isVar;
if (node.Type == NonTerminatorType.VarParameter)
{
isVar = true;
}
else if (node.Type == NonTerminatorType.ValueParameter)
{
isVar = false;
}
else
{
throw new InvalidOperationException();
}
return new Parameter { Children = children, IsVar = isVar };
}
}

View File

@ -0,0 +1,54 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ParameterList : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ParameterList;
public bool IsRecursive { get; private init; }
/// <summary>
/// 声明的参数列表
/// </summary>
public IEnumerable<Parameter> Parameters => GetParameters();
public static ParameterList Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 1)
{
isRecursive = false;
}
else if (children.Count == 3)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new ParameterList { Children = children, IsRecursive = isRecursive };
}
private IEnumerable<Parameter> GetParameters()
{
ParameterList list = this;
while (true)
{
if (list.IsRecursive)
{
yield return list.Children[2].Convert<Parameter>();
list = list.Children[0].Convert<ParameterList>();
}
else
{
yield return list.Children[0].Convert<Parameter>();
break;
}
}
}
}

View File

@ -0,0 +1,69 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class Period : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Period;
public bool IsRecursive { get; private init; }
/// <summary>
/// 数组上下界列表
/// </summary>
public IEnumerable<(NumberSemanticToken, NumberSemanticToken)> Ranges => GetRanges();
public static Period Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 3)
{
isRecursive = false;
}
else if (children.Count == 5)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new Period { Children = children, IsRecursive = isRecursive };
}
private (NumberSemanticToken, NumberSemanticToken) GetRange()
{
if (IsRecursive)
{
return ((NumberSemanticToken)Children[2].Convert<TerminatedSyntaxNode>().Token,
(NumberSemanticToken)Children[4].Convert<TerminatedSyntaxNode>().Token);
}
else
{
return ((NumberSemanticToken)Children[0].Convert<TerminatedSyntaxNode>().Token,
(NumberSemanticToken)Children[2].Convert<TerminatedSyntaxNode>().Token);
}
}
private IEnumerable<(NumberSemanticToken, NumberSemanticToken)> GetRanges()
{
Period period = this;
while (true)
{
if (period.IsRecursive)
{
yield return period.GetRange();
period = period.Children[0].Convert<Period>();
}
else
{
yield return period.GetRange();
break;
}
}
}
}

View File

@ -0,0 +1,32 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class ProcedureCall : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ProcedureCall;
public IdentifierSemanticToken ProcedureId
=> (IdentifierSemanticToken)Children[0].Convert<TerminatedSyntaxNode>().Token;
public IEnumerable<Expression> Arguments => GetArguments();
public static ProcedureCall Create(List<SyntaxNodeBase> children)
{
return new ProcedureCall { Children = children };
}
private IEnumerable<Expression> GetArguments()
{
if (Children.Count == 1)
{
yield break;
}
foreach (Expression expression in Children[2].Convert<ExpressionList>().Expressions)
{
yield return expression;
}
}
}

View File

@ -0,0 +1,33 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ProgramBody : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ProgramBody;
/// <summary>
/// 常量声明
/// </summary>
public ConstDeclarations ConstDeclarations => Children[0].Convert<ConstDeclarations>();
/// <summary>
/// 变量声明
/// </summary>
public VarDeclarations VarDeclarations => Children[1].Convert<VarDeclarations>();
/// <summary>
/// 子程序声明
/// </summary>
public SubprogramDeclarations SubprogramDeclarations => Children[2].Convert<SubprogramDeclarations>();
/// <summary>
/// 语句声明
/// </summary>
public CompoundStatement CompoundStatement => Children[3].Convert<CompoundStatement>();
public static ProgramBody Create(List<SyntaxNodeBase> children)
{
return new ProgramBody { Children = children };
}
}

View File

@ -0,0 +1,40 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class ProgramHead : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ProgramHead;
/// <summary>
/// 程序名称
/// </summary>
public IdentifierSemanticToken ProgramName
=> (IdentifierSemanticToken)Children[1].Convert<TerminatedSyntaxNode>().Token;
/// <summary>
/// 暂时意义不明的标识符列表
/// https://wiki.freepascal.org/Program_Structure/zh_CN
/// TODO: 查阅资料
/// </summary>
public IEnumerable<IdentifierSemanticToken> FileList => GetFileList();
public static ProgramHead Create(List<SyntaxNodeBase> children)
{
return new ProgramHead { Children = children };
}
private IEnumerable<IdentifierSemanticToken> GetFileList()
{
if (Children.Count == 2)
{
yield break;
}
foreach (IdentifierSemanticToken token in Children[3].Convert<IdentifierList>().Identifiers)
{
yield return token;
}
}
}

View File

@ -0,0 +1,23 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ProgramStruct : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ProgramStruct;
/// <summary>
/// 程序头
/// </summary>
public ProgramHead Head => Children[0].Convert<ProgramHead>();
/// <summary>
/// 程序体
/// </summary>
public ProgramBody Body => Children[2].Convert<ProgramBody>();
public static ProgramStruct Create(List<SyntaxNodeBase> children)
{
return new ProgramStruct { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class RelationOperator : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.RelationOperator;
public static RelationOperator Create(List<SyntaxNodeBase> children)
{
return new RelationOperator { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class SimpleExpression : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.SimpleExpression;
public static SimpleExpression Create(List<SyntaxNodeBase> children)
{
return new SimpleExpression { Children = children };
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Statement : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Statement;
public static Statement Create(List<SyntaxNodeBase> children)
{
return new Statement { Children = children };
}
}

View File

@ -0,0 +1,51 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class StatementList : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.StatementList;
public bool IsRecursive { get; private init; }
public IEnumerable<Statement> Statements => GetStatements();
public static StatementList Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 1)
{
isRecursive = false;
}
else if (children.Count == 3)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new StatementList { Children = children, IsRecursive = isRecursive };
}
private IEnumerable<Statement> GetStatements()
{
StatementList list = this;
while (true)
{
if (list.IsRecursive)
{
yield return list.Children[2].Convert<Statement>();
list = list.Children[0].Convert<StatementList>();
}
else
{
yield return list.Children[0].Convert<Statement>();
break;
}
}
}
}

View File

@ -0,0 +1,23 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Subprogram : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Subprogram;
/// <summary>
/// 子程序头部
/// </summary>
public SubprogramHead Head => Children[0].Convert<SubprogramHead>();
/// <summary>
/// 子程序体
/// </summary>
public SubprogramBody Body => Children[2].Convert<SubprogramBody>();
public static Subprogram Create(List<SyntaxNodeBase> children)
{
return new Subprogram { Children = children };
}
}

View File

@ -0,0 +1,28 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class SubprogramBody : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.SubprogramBody;
/// <summary>
/// 常量声明部分
/// </summary>
public ConstDeclarations ConstDeclarations => Children[0].Convert<ConstDeclarations>();
/// <summary>
/// 变量声明部分
/// </summary>
public VarDeclarations VarDeclarations => Children[1].Convert<VarDeclarations>();
/// <summary>
/// 语句声明部分
/// </summary>
public CompoundStatement CompoundStatement => Children[2].Convert<CompoundStatement>();
public static SubprogramBody Create(List<SyntaxNodeBase> children)
{
return new SubprogramBody() { Children = children };
}
}

View File

@ -0,0 +1,34 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class SubprogramDeclarations : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.SubprogramDeclarations;
/// <summary>
/// 声明的子程序列表
/// </summary>
public IEnumerable<Subprogram> Subprograms => GetSubprograms();
public static SubprogramDeclarations Create(List<SyntaxNodeBase> children)
{
return new SubprogramDeclarations { Children = children };
}
private IEnumerable<Subprogram> GetSubprograms()
{
SubprogramDeclarations declarations = this;
while (true)
{
if (declarations.Children.Count == 0)
{
yield break;
}
yield return declarations.Children[1].Convert<Subprogram>();
declarations = declarations.Children[0].Convert<SubprogramDeclarations>();
}
}
}

View File

@ -0,0 +1,45 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class SubprogramHead : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.SubprogramHead;
public bool IsProcedure { get; private init; }
/// <summary>
/// 子程序的名称
/// </summary>
public IdentifierSemanticToken SubprogramName =>
(IdentifierSemanticToken)Children[1].Convert<TerminatedSyntaxNode>().Token;
/// <summary>
/// 子程序的参数
/// </summary>
public IEnumerable<Parameter> Parameters => Children[2].Convert<FormalParameter>().Parameters;
public static SubprogramHead Create(List<SyntaxNodeBase> children)
{
bool isProcedure;
TerminatedSyntaxNode node = children[0].Convert<TerminatedSyntaxNode>();
KeywordSemanticToken token = (KeywordSemanticToken)node.Token;
if (token.KeywordType == KeywordType.Procedure)
{
isProcedure = true;
}
else if (token.KeywordType == KeywordType.Function)
{
isProcedure = false;
}
else
{
throw new InvalidOperationException();
}
return new SubprogramHead { Children = children, IsProcedure = isProcedure };
}
}

View File

@ -0,0 +1,107 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public abstract class SyntaxNodeBase
{
public abstract bool IsTerminated { get; }
public T Convert<T>() where T : SyntaxNodeBase
{
T? result = this as T;
if (result is null)
{
throw new InvalidOperationException("Can't cast into target SyntaxNode");
}
return result;
}
public static SyntaxNodeBase Create(SemanticToken token)
{
return new TerminatedSyntaxNode { Token = token };
}
public static SyntaxNodeBase Create(NonTerminatorType type, List<SyntaxNodeBase> children)
{
switch (type)
{
case NonTerminatorType.ProgramStruct:
return ProgramStruct.Create(children);
case NonTerminatorType.ProgramHead:
return ProgramHead.Create(children);
case NonTerminatorType.ProgramBody:
return ProgramBody.Create(children);
case NonTerminatorType.IdentifierList:
return IdentifierList.Create(children);
case NonTerminatorType.VarDeclarations:
return VarDeclarations.Create(children);
case NonTerminatorType.SubprogramDeclarations:
return SubprogramDeclarations.Create(children);
case NonTerminatorType.CompoundStatement:
return CompoundStatement.Create(children);
case NonTerminatorType.ConstValue:
return ConstValue.Create(children);
case NonTerminatorType.ConstDeclaration:
return ConstDeclaration.Create(children);
case NonTerminatorType.ConstDeclarations:
return ConstDeclarations.Create(children);
case NonTerminatorType.VarDeclaration:
return VarDeclaration.Create(children);
case NonTerminatorType.Type:
return TypeSyntaxNode.Create(children);
case NonTerminatorType.BasicType:
return BasicType.Create(children);
case NonTerminatorType.Period:
return Period.Create(children);
case NonTerminatorType.Subprogram:
return Subprogram.Create(children);
case NonTerminatorType.SubprogramHead:
return SubprogramHead.Create(children);
case NonTerminatorType.SubprogramBody:
return SubprogramBody.Create(children);
case NonTerminatorType.FormalParameter:
return FormalParameter.Create(children);
case NonTerminatorType.ParameterList:
return ParameterList.Create(children);
case NonTerminatorType.Parameter:
return Parameter.Create(children);
case NonTerminatorType.VarParameter:
return VarParameter.Create(children);
case NonTerminatorType.ValueParameter:
return ValueParameter.Create(children);
case NonTerminatorType.StatementList:
return StatementList.Create(children);
case NonTerminatorType.Statement:
return Statement.Create(children);
case NonTerminatorType.Variable:
return Variable.Create(children);
case NonTerminatorType.Expression:
return Expression.Create(children);
case NonTerminatorType.ProcedureCall:
return ProcedureCall.Create(children);
case NonTerminatorType.ElsePart:
return ElsePart.Create(children);
case NonTerminatorType.ExpressionList:
return ExpressionList.Create(children);
case NonTerminatorType.SimpleExpression:
return SimpleExpression.Create(children);
case NonTerminatorType.Term:
return Term.Create(children);
case NonTerminatorType.Factor:
return Factor.Create(children);
case NonTerminatorType.AddOperator:
return AddOperator.Create(children);
case NonTerminatorType.MultiplyOperator:
return MultiplyOperator.Create(children);
case NonTerminatorType.RelationOperator:
return RelationOperator.Create(children);
case NonTerminatorType.IdVarPart:
return IdentifierVarPart.Create(children);
default:
throw new InvalidOperationException();
}
}
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class Term : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Term;
public static Term Create(List<SyntaxNodeBase> children)
{
return new Term { Children = children };
}
}

View File

@ -0,0 +1,10 @@
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class TerminatedSyntaxNode : SyntaxNodeBase
{
public override bool IsTerminated => true;
public required SemanticToken Token { get; init; }
}

View File

@ -0,0 +1,13 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class TypeSyntaxNode : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Type;
public static TypeSyntaxNode Create(List<SyntaxNodeBase> children)
{
return new TypeSyntaxNode { Children = children };
}
}

View File

@ -0,0 +1,23 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class ValueParameter : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.ValueParameter;
/// <summary>
/// 声明的变量列表
/// </summary>
public IdentifierList IdentifierList => Children[0].Convert<IdentifierList>();
/// <summary>
/// 声明的变量类型
/// </summary>
public BasicType BasicType => Children[2].Convert<BasicType>();
public static ValueParameter Create(List<SyntaxNodeBase> children)
{
return new ValueParameter { Children = children };
}
}

View File

@ -0,0 +1,47 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class VarDeclaration : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.VarDeclaration;
public bool IsRecursive { get; private init; }
/// <summary>
/// 声明的变量
/// </summary>
public (IdentifierList, TypeSyntaxNode) Variable => GetVariable();
private (IdentifierList, TypeSyntaxNode) GetVariable()
{
if (IsRecursive)
{
return (Children[2].Convert<IdentifierList>(), Children[4].Convert<TypeSyntaxNode>());
}
else
{
return (Children[0].Convert<IdentifierList>(), Children[2].Convert<TypeSyntaxNode>());
}
}
public static VarDeclaration Create(List<SyntaxNodeBase> children)
{
bool isRecursive;
if (children.Count == 3)
{
isRecursive = false;
}
else if (children.Count == 5)
{
isRecursive = true;
}
else
{
throw new InvalidOperationException();
}
return new VarDeclaration { Children = children, IsRecursive = isRecursive };
}
}

View File

@ -0,0 +1,42 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class VarDeclarations : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.VarDeclarations;
/// <summary>
/// 声明的变量列表
/// </summary>
public IEnumerable<(IdentifierList, TypeSyntaxNode)> Variables => EnumerateVariables();
public static VarDeclarations Create(List<SyntaxNodeBase> children)
{
return new VarDeclarations { Children = children };
}
private IEnumerable<(IdentifierList, TypeSyntaxNode)> EnumerateVariables()
{
if (Children.Count == 0)
{
yield break;
}
VarDeclaration declaration = Children[1].Convert<VarDeclaration>();
while (true)
{
yield return declaration.Variable;
if (declaration.IsRecursive)
{
declaration = declaration.Children[0].Convert<VarDeclaration>();
}
else
{
break;
}
}
}
}

View File

@ -0,0 +1,15 @@
using Canon.Core.Enums;
namespace Canon.Core.SyntaxNodes;
public class VarParameter : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.VarParameter;
public ValueParameter ValueParameter => Children[1].Convert<ValueParameter>();
public static VarParameter Create(List<SyntaxNodeBase> children)
{
return new VarParameter { Children = children };
}
}

View File

@ -0,0 +1,20 @@
using Canon.Core.Enums;
using Canon.Core.LexicalParser;
namespace Canon.Core.SyntaxNodes;
public class Variable : NonTerminatedSyntaxNode
{
public override NonTerminatorType Type => NonTerminatorType.Variable;
/// <summary>
/// 变量的名称
/// </summary>
public IdentifierSemanticToken Identifier =>
(IdentifierSemanticToken)Children[0].Convert<TerminatedSyntaxNode>().Token;
public static Variable Create(List<SyntaxNodeBase> children)
{
return new Variable { Children = children };
}
}

View File

@ -1,413 +0,0 @@
using Canon.Core.Abstractions;
using Canon.Core.Enums;
using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
namespace Canon.Tests.GrammarParserTests;
public class PascalGrammarConstPart
{
private static readonly Dictionary<NonTerminator, List<List<TerminatorBase>>> s_pascalGrammar = new()
{
{
// ProgramStart -> ProgramBody
new NonTerminator(NonTerminatorType.StartNonTerminator), [
[new NonTerminator(NonTerminatorType.ProgramBody)]
]
},
{
// ProgramBody -> ConstDeclarations
new NonTerminator(NonTerminatorType.ProgramBody), [
[
new NonTerminator(NonTerminatorType.ConstDeclarations),
// new NonTerminator(NonTerminatorType.VarDeclarations),
// new NonTerminator(NonTerminatorType.SubprogramDeclarations),
// new NonTerminator(NonTerminatorType.CompoundStatement)
]
]
},
{
// ConstDeclarations -> ε | const ConstDeclaration ;
new NonTerminator(NonTerminatorType.ConstDeclarations), [
[
Terminator.EmptyTerminator,
],
[
new Terminator(KeywordType.Const),
new NonTerminator(NonTerminatorType.ConstDeclaration),
new Terminator(DelimiterType.Semicolon)
]
]
},
{
// ConstDeclaration -> id = ConstValue | ConstDeclaration ; id = ConstValue
new NonTerminator(NonTerminatorType.ConstDeclaration), [
[
Terminator.IdentifierTerminator,
new Terminator(OperatorType.Equal),
new NonTerminator(NonTerminatorType.ConstValue)
],
[
new NonTerminator(NonTerminatorType.ConstDeclaration),
new Terminator(DelimiterType.Semicolon),
Terminator.IdentifierTerminator,
new Terminator(OperatorType.Equal),
new NonTerminator(NonTerminatorType.ConstValue)
]
]
},
{
// ConstValue -> +num | -num | num | 'letter'
new NonTerminator(NonTerminatorType.ConstValue), [
[
new Terminator(OperatorType.Plus), Terminator.NumberTerminator
],
[
new Terminator(OperatorType.Minus), Terminator.NumberTerminator,
],
[
Terminator.NumberTerminator,
],
[
new Terminator(DelimiterType.SingleQuotation),
Terminator.CharacterTerminator,
new Terminator(DelimiterType.SingleQuotation),
]
]
}
};
[Fact]
public void AstTestFirst()
{
GrammarBuilder builder = new()
{
Generators = s_pascalGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
GrammarParserBase grammar = builder.Build().ToGrammarParser();
// const a = +5;
List<SemanticToken> tokens =
[
new KeywordSemanticToken
{
CharacterPos = 0, KeywordType = KeywordType.Const, LinePos = 0, LiteralValue = "const"
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "a" },
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "=", OperatorType = OperatorType.Equal
},
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "+", OperatorType = OperatorType.Plus
},
new NumberSemanticToken
{
CharacterPos = 0, LinePos = 0, LiteralValue = "5", NumberType = NumberType.Integer
},
new DelimiterSemanticToken
{
CharacterPos = 0, DelimiterType = DelimiterType.Semicolon, LinePos = 0, LiteralValue = ";"
},
SemanticToken.End
];
// ProgramBody
SyntaxNode root = grammar.Analyse(tokens);
Assert.Equal(NonTerminatorType.ProgramBody, root.GetNonTerminatorType());
Assert.Single(root.Children);
Assert.Equal(10, root.Count());
// ConstDeclarations
root = root.Children[0];
Assert.Equal(NonTerminatorType.ConstDeclarations, root.GetNonTerminatorType());
Assert.Equal(3, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Keyword)
{
KeywordSemanticToken token = (KeywordSemanticToken)node.GetSemanticToken();
return token.KeywordType == KeywordType.Const;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Delimiter)
{
DelimiterSemanticToken token = (DelimiterSemanticToken)node.GetSemanticToken();
return token.DelimiterType == DelimiterType.Semicolon;
}
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration)
{
return true;
}
return false;
});
// ConstDeclaration
root = root.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstDeclaration, root.GetNonTerminatorType());
Assert.Equal(3, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Identifier)
{
return true;
}
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstValue)
{
return true;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Operator)
{
OperatorSemanticToken token = (OperatorSemanticToken)node.GetSemanticToken();
return token.OperatorType == OperatorType.Equal;
}
return false;
});
// ConstValue
root = root.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstValue
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstValue, root.GetNonTerminatorType());
Assert.Equal(2, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Operator)
{
OperatorSemanticToken token = (OperatorSemanticToken)node.GetSemanticToken();
return token.OperatorType == OperatorType.Plus;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Number)
{
return true;
}
return false;
});
}
[Fact]
public void AstTestSecond()
{
GrammarBuilder builder = new()
{
Generators = s_pascalGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
GrammarParserBase grammar = builder.Build().ToGrammarParser();
// const a = 5; McCafe = 'Under Pressure (Queen & D. Bowie)' ;
List<SemanticToken> tokens =
[
new KeywordSemanticToken
{
CharacterPos = 0, KeywordType = KeywordType.Const, LinePos = 0, LiteralValue = "const"
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "a" },
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "=", OperatorType = OperatorType.Equal
},
new NumberSemanticToken
{
CharacterPos = 0, LinePos = 0, LiteralValue = "5", NumberType = NumberType.Integer
},
new DelimiterSemanticToken
{
CharacterPos = 0, DelimiterType = DelimiterType.Semicolon, LinePos = 0, LiteralValue = ";"
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "McCafe" },
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "=", OperatorType = OperatorType.Equal
},
new DelimiterSemanticToken
{
CharacterPos = 0, DelimiterType = DelimiterType.SingleQuotation, LinePos = 0, LiteralValue = "'"
},
new CharacterSemanticToken
{
CharacterPos = 0, LinePos = 0, LiteralValue = "Under Pressure (Queen & D. Bowie)"
},
new DelimiterSemanticToken
{
CharacterPos = 0, DelimiterType = DelimiterType.SingleQuotation, LinePos = 0, LiteralValue = "'"
},
new DelimiterSemanticToken
{
CharacterPos = 0, DelimiterType = DelimiterType.Semicolon, LinePos = 0, LiteralValue = ";"
},
SemanticToken.End
];
// 分析树见文档
// ProgramBody
SyntaxNode root = grammar.Analyse(tokens);
Assert.Equal(NonTerminatorType.ProgramBody, root.GetNonTerminatorType());
Assert.Single(root.Children);
Assert.Equal(17, root.Count());
// ConstDeclarations
root = root.Children[0];
Assert.Equal(NonTerminatorType.ConstDeclarations, root.GetNonTerminatorType());
Assert.Equal(3, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Keyword)
{
KeywordSemanticToken token = (KeywordSemanticToken)node.GetSemanticToken();
return token.KeywordType == KeywordType.Const;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Delimiter)
{
DelimiterSemanticToken token = (DelimiterSemanticToken)node.GetSemanticToken();
return token.DelimiterType == DelimiterType.Semicolon;
}
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration)
{
return true;
}
return false;
});
// ConstDeclaration layer3
root = root.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstDeclaration, root.GetNonTerminatorType());
Assert.Equal(5, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration)
{
return true;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Delimiter)
{
DelimiterSemanticToken token = (DelimiterSemanticToken)node.GetSemanticToken();
return token.DelimiterType == DelimiterType.Semicolon;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Identifier)
{
return true;
}
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstValue)
{
return true;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Operator)
{
OperatorSemanticToken token = (OperatorSemanticToken)node.GetSemanticToken();
return token.OperatorType == OperatorType.Equal;
}
return false;
});
// ConstDeclaration layer4
SyntaxNode constDeclarationLayer4 = root.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstDeclaration
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstDeclaration, constDeclarationLayer4.GetNonTerminatorType());
Assert.Equal(3, constDeclarationLayer4.Children.Count);
Assert.Contains(constDeclarationLayer4.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Identifier)
{
return true;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Operator)
{
OperatorSemanticToken token = (OperatorSemanticToken)node.GetSemanticToken();
return token.OperatorType == OperatorType.Equal;
}
if (!node.IsTerminated && node.GetNonTerminatorType() == NonTerminatorType.ConstValue)
{
return true;
}
return false;
});
// ConstValue layer4
SyntaxNode constValueLayer4 = root.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstValue
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstValue, constValueLayer4.GetNonTerminatorType());
Assert.Equal(3, constValueLayer4.Children.Count);
Assert.Contains(constValueLayer4.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Delimiter)
{
DelimiterSemanticToken token = (DelimiterSemanticToken)node.GetSemanticToken();
return token.DelimiterType == DelimiterType.SingleQuotation;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Character)
{
return true;
}
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Delimiter)
{
DelimiterSemanticToken token = (DelimiterSemanticToken)node.GetSemanticToken();
return token.DelimiterType == DelimiterType.SingleQuotation;
}
return false;
});
// ConstValue layer5
SyntaxNode constValueLayer5 = constDeclarationLayer4.Children.Where(child =>
!child.IsTerminated && child.GetNonTerminatorType() == NonTerminatorType.ConstValue
).ElementAt(0);
Assert.Equal(NonTerminatorType.ConstValue, constValueLayer5.GetNonTerminatorType());
Assert.Single(constValueLayer5.Children);
Assert.Contains(constValueLayer5.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Number)
{
return true;
}
return false;
});
}
}

View File

@ -1,14 +1,16 @@
using Canon.Core.Abstractions; using Canon.Core.Abstractions;
using Canon.Core.Enums; using Canon.Core.Enums;
using Canon.Core.GrammarParser; using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
using Canon.Core.SyntaxNodes;
namespace Canon.Tests.GrammarParserTests; namespace Canon.Tests.GrammarParserTests;
public partial class PascalGrammarTests public class PascalGrammarTests
{ {
private readonly GrammarBuilder _builder = new() private readonly GrammarBuilder _builder = new()
{ {
Generators = s_pascalGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator) Generators = PascalGrammar.Grammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
}; };
private readonly GrammarParserBase _parser; private readonly GrammarParserBase _parser;
@ -24,4 +26,21 @@ public partial class PascalGrammarTests
{ {
Assert.NotNull(_parser); Assert.NotNull(_parser);
} }
[Fact]
public void DoNothingTest()
{
const string program = """
program DoNothing;
begin
end.
""";
Lexer lexer = new(program);
List<SemanticToken> tokens = lexer.Tokenize();
tokens.Add(SemanticToken.End);
ProgramStruct root = _parser.Analyse(tokens).Convert<ProgramStruct>();
Assert.Equal("DoNothing", root.Head.ProgramName.LiteralValue);
}
} }

View File

@ -1,7 +1,5 @@
using Canon.Core.Abstractions; using Canon.Core.Enums;
using Canon.Core.Enums;
using Canon.Core.GrammarParser; using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
namespace Canon.Tests.GrammarParserTests; namespace Canon.Tests.GrammarParserTests;
@ -109,99 +107,4 @@ public class SimpleGrammarTests
Assert.Contains(new Terminator(DelimiterType.LeftParenthesis), grammar.BeginState.Transformer.Keys); Assert.Contains(new Terminator(DelimiterType.LeftParenthesis), grammar.BeginState.Transformer.Keys);
Assert.Contains(Terminator.IdentifierTerminator, grammar.BeginState.Transformer.Keys); Assert.Contains(Terminator.IdentifierTerminator, grammar.BeginState.Transformer.Keys);
} }
[Fact]
public void AnalyseSingleSentenceTest()
{
GrammarBuilder builder = new()
{
Generators = s_simpleGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
// n + n
List<SemanticToken> tokens =
[
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "n" },
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "+", OperatorType = OperatorType.Plus
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "n" },
SemanticToken.End
];
// 分析树为
// E
// |
// /\
// / | \
// E + T
// | |
// T F
// | |
// F n
// |
// n
SyntaxNode root = parser.Analyse(tokens);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());
Assert.Equal(3, root.Children.Count);
Assert.Contains(root.Children, node =>
{
if (node.IsTerminated && node.GetSemanticToken().TokenType == SemanticTokenType.Operator)
{
OperatorSemanticToken token = (OperatorSemanticToken)node.GetSemanticToken();
return token.OperatorType == OperatorType.Plus;
}
return false;
});
Assert.Equal(9, root.Count());
}
[Fact]
public void AnalyseComplexSentenceTest()
{
GrammarBuilder builder = new()
{
Generators = s_simpleGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
// (n + n) * n
List<SemanticToken> tokens =
[
new DelimiterSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "(", DelimiterType = DelimiterType.LeftParenthesis
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "n" },
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "+", OperatorType = OperatorType.Plus
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "n" },
new DelimiterSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = ")", DelimiterType = DelimiterType.RightParenthesis
},
new OperatorSemanticToken
{
LinePos = 0, CharacterPos = 0, LiteralValue = "*", OperatorType = OperatorType.Multiply
},
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "n" },
SemanticToken.End
];
SyntaxNode root = parser.Analyse(tokens);
Assert.Equal(18, root.Count());
Assert.False(root.IsTerminated);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());
Assert.Single(root.Children);
}
} }

View File

@ -1,7 +1,6 @@
using Canon.Core.Abstractions; using Canon.Core.Abstractions;
using Canon.Core.Enums; using Canon.Core.Enums;
using Canon.Core.GrammarParser; using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
using Xunit.Abstractions; using Xunit.Abstractions;
namespace Canon.Tests.GrammarParserTests; namespace Canon.Tests.GrammarParserTests;
@ -167,29 +166,4 @@ public class SimpleGrammarWithEmptyTests(ITestOutputHelper testOutputHelper)
Assert.Single(transformer1.ReduceTable); Assert.Single(transformer1.ReduceTable);
Assert.Contains(new NonTerminator(NonTerminatorType.ProgramStruct),transformer1.ShiftTable); Assert.Contains(new NonTerminator(NonTerminatorType.ProgramStruct),transformer1.ShiftTable);
} }
[Fact]
public void AnalyseSingleSentenceTest()
{
GrammarBuilder builder = new()
{
Generators = s_simpleGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
List<SemanticToken> tokens =
[
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "a" },
new IdentifierSemanticToken { LinePos = 0, CharacterPos = 0, LiteralValue = "a" },
SemanticToken.End
];
SyntaxNode root = parser.Analyse(tokens);
Assert.False(root.IsTerminated);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());
Assert.Equal(7, root.Count());
}
} }

View File

@ -1,6 +1,4 @@
using Canon.Core.GrammarParser; namespace Canon.Tests;
namespace Canon.Tests;
public static class Utils public static class Utils
{ {
@ -15,25 +13,4 @@ public static class Utils
return list; return list;
} }
/// <summary>
/// 验证两棵语法树一致
/// </summary>
/// <param name="a">一棵语法树</param>
/// <param name="b">另一棵语法树</param>
public static void CheckSyntaxRoot(SyntaxNode a, SyntaxNode b)
{
int length = a.Count();
Assert.Equal(length, b.Count());
using IEnumerator<SyntaxNode> aIter = a.GetEnumerator(), bIter = b.GetEnumerator();
for (int i = 0; i < length; i++)
{
Assert.True(aIter.MoveNext());
Assert.True(bIter.MoveNext());
Assert.Equal(aIter.Current, bIter.Current);
}
}
} }