post-guard/Canon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 添加语法分析基类抽象 (#8)

Browse Source 
增加语法分析基类和状态转换接口抽象，为直接生成语法分析器做准备，同时也提前释放一些大对象，降低内存消耗。

Reviewed-on: PostGuard/Canon#8
This commit is contained in:
jackfiled
2024-03-13 16:41:44 +08:00
parent e191c1e077
commit bd3db1b7b7
8 changed files with 224 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Canon.Tests/GrammarParserTests/SimpleGrammarTests.cs
View File

@@ -1,4 +1,5 @@
using Canon.Core.Enums;
using Canon.Core.Abstractions;
using Canon.Core.Enums;
using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
@@ -118,6 +119,7 @@ public class SimpleGrammarTests
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
// n + n
List<SemanticToken> tokens =
[
@@ -142,7 +144,7 @@ public class SimpleGrammarTests
// F n
// |
// n
SyntaxNode root = grammar.Analyse(tokens);
SyntaxNode root = parser.Analyse(tokens);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());
Assert.Equal(3, root.Children.Count);
Assert.Contains(root.Children, node =>
@@ -168,6 +170,8 @@ public class SimpleGrammarTests
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
// (n + n) * n
List<SemanticToken> tokens =
[
@@ -193,7 +197,8 @@ public class SimpleGrammarTests
SemanticToken.End
];
SyntaxNode root = grammar.Analyse(tokens);
SyntaxNode root = parser.Analyse(tokens);
Assert.Equal(18, root.Count());
Assert.False(root.IsTerminated);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());

23
Canon.Tests/GrammarParserTests/SimpleGrammarWithEmptyTests.cs
View File

@@ -1,4 +1,5 @@
using Canon.Core.Enums;
using Canon.Core.Abstractions;
using Canon.Core.Enums;
using Canon.Core.GrammarParser;
using Canon.Core.LexicalParser;
using Xunit.Abstractions;
@@ -151,6 +152,23 @@ public class SimpleGrammarWithEmptyTests(ITestOutputHelper testOutputHelper)
_testOutputHelper.WriteLine(state5.ToString());
}
[Fact]
public void ParserTest()
{
GrammarBuilder builder = new()
{
Generators = s_simpleGrammar, Begin = new NonTerminator(NonTerminatorType.StartNonTerminator)
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
ITransformer transformer1 = parser.BeginTransformer;
Assert.Equal(3, transformer1.ShiftTable.Count);
Assert.Single(transformer1.ReduceTable);
Assert.Contains(new NonTerminator(NonTerminatorType.ProgramStruct),transformer1.ShiftTable);
}
[Fact]
public void AnalyseSingleSentenceTest()
{
@@ -160,6 +178,7 @@ public class SimpleGrammarWithEmptyTests(ITestOutputHelper testOutputHelper)
};
Grammar grammar = builder.Build();
GrammarParserBase parser = grammar.ToGrammarParser();
List<SemanticToken> tokens =
[
@@ -168,7 +187,7 @@ public class SimpleGrammarWithEmptyTests(ITestOutputHelper testOutputHelper)
SemanticToken.End
];
SyntaxNode root = grammar.Analyse(tokens);
SyntaxNode root = parser.Analyse(tokens);
Assert.False(root.IsTerminated);
Assert.Equal(NonTerminatorType.ProgramStruct, root.GetNonTerminatorType());

60
Canon.Tests/GrammarParserTests/TerminatorTests.cs
View File

@@ -27,11 +27,10 @@ public class TerminatorTests
public void TerminatorAndKeywordSemanticTokenTest()
{
Terminator keywordTerminator = new(KeywordType.Array);
LinkedList<char> keywordContent = Utils.GetLinkedList("array [3..9] of integer");
Assert.True(KeywordSemanticToken.TryParse(0, 0, keywordContent.First!,
out KeywordSemanticToken? keywordSemanticToken));
Assert.NotNull(keywordSemanticToken);
KeywordSemanticToken keywordSemanticToken = new()
{
LinePos = 0, CharacterPos = 0, KeywordType = KeywordType.Array, LiteralValue = "array"
};
Assert.True(keywordTerminator == keywordSemanticToken);
}
@@ -39,11 +38,10 @@ public class TerminatorTests
public void TerminatorAndDelimiterSemanticTokenTest()
{
Terminator terminator = new(DelimiterType.Period);
LinkedList<char> content = Utils.GetLinkedList(".");
Assert.True(DelimiterSemanticToken.TryParse(0, 0, content.First!,
out DelimiterSemanticToken? token));
Assert.NotNull(token);
DelimiterSemanticToken token = new()
{
LinePos = 0, CharacterPos = 0, DelimiterType = DelimiterType.Period, LiteralValue = "."
};
Assert.True(token == terminator);
}
@@ -51,44 +49,10 @@ public class TerminatorTests
public void TerminatorAndOperatorSemanticTokenTest()
{
Terminator terminator = new(OperatorType.GreaterEqual);
LinkedList<char> content = Utils.GetLinkedList(">=");
Assert.True(OperatorSemanticToken.TryParse(0, 0, content.First!,
out OperatorSemanticToken? token));
Assert.NotNull(token);
OperatorSemanticToken token = new()
{
LinePos = 0, CharacterPos = 0, OperatorType = OperatorType.GreaterEqual, LiteralValue = ">="
};
Assert.True(token == terminator);
}
[Fact]
public void TerminatorAndNumberSemanticTokenTest()
{
LinkedList<char> content = Utils.GetLinkedList("123");
Assert.True(NumberSemanticToken.TryParse(0, 0, content.First!,
out NumberSemanticToken? token));
Assert.NotNull(token);
Assert.True(Terminator.NumberTerminator == token);
}
[Fact]
public void TerminatorAndCharacterSemanticTokenTest()
{
LinkedList<char> content = Utils.GetLinkedList("'a'");
Assert.True(CharacterSemanticToken.TryParse(0, 0, content.First!,
out CharacterSemanticToken? token));
Assert.NotNull(token);
Assert.True(Terminator.CharacterTerminator == token);
}
[Fact]
public void TerminatorAndIdentifierSemanticTokenTest()
{
LinkedList<char> content = Utils.GetLinkedList("gcd");
Assert.True(IdentifierSemanticToken.TryParse(0, 0, content.First!,
out IdentifierSemanticToken? token));
Assert.NotNull(token);
Assert.True(Terminator.IdentifierTerminator == token);
}
}