jackfiled/CanonSharp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Parser Combinator库和词法分析器 (#2)
All checks were successful
Run unit test / Unit-Test (push) Successful in 41s

Browse Source 
Reviewed-on: https://git.bupt-hpc.cn/jackfiled/CanonSharp/pulls/2
Co-authored-by: jackfiled <xcrenchangjun@outlook.com>
Co-committed-by: jackfiled <xcrenchangjun@outlook.com>
This commit is contained in:
jackfiled
2024-08-13 14:46:11 +08:00
committed by 任昌骏
parent 57c31ec435
commit 3ed8bf5d36
68 changed files with 3133 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
CanonSharp.Tests/ScannerTest/LexicalParserTests.cs Normal file
View File

@@ -0,0 +1,167 @@
using CanonSharp.Common.Scanner;
using CanonSharp.Tests.Utils;
namespace CanonSharp.Tests.ScannerTest;
public class LexicalParserTests : LexicalTestBase
{
[Fact]
public void LexicalParserTest1()
{
const string pascalProgram = """
program HelloWorld;
var
message: char;
begin
message := 'h';
writeln(message);
end.
""";
ValidateLexicalTokens(LexicalScanner.PascalParser(), pascalProgram, [
(LexicalTokenType.Keyword, "program"),
(LexicalTokenType.Identifier, "HelloWorld"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "var"),
(LexicalTokenType.Identifier, "message"),
(LexicalTokenType.Delimiter, ":"),
(LexicalTokenType.Keyword, "char"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "begin"),
(LexicalTokenType.Identifier, "message"),
(LexicalTokenType.Operator, ":="),
(LexicalTokenType.Character, "h"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Identifier, "writeln"),
(LexicalTokenType.Delimiter, "("),
(LexicalTokenType.Identifier, "message"),
(LexicalTokenType.Delimiter, ")"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "end"),
(LexicalTokenType.Delimiter, ".")
]);
}
[Fact]
public void LexicalParserTest2()
{
const string program = """
program main;
var
ab: integer;
begin
ab := 3;
write(ab);
end.
""";
ValidateLexicalTokens(LexicalScanner.PascalParser(), program, [
(LexicalTokenType.Keyword, "program"),
(LexicalTokenType.Identifier, "main"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "var"),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Delimiter, ":"),
(LexicalTokenType.Keyword, "integer"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "begin"),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Operator, ":="),
(LexicalTokenType.ConstInteger, "3"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Identifier, "write"),
(LexicalTokenType.Delimiter, "("),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Delimiter, ")"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "end"),
(LexicalTokenType.Delimiter, ".")
]);
}
[Fact]
public void LexicalParserTest3()
{
const string pascalProgram = """
{test}
program main;
var
ab, ba: integer;
begin
ab := 3;
ba := 5;
ab := 5;
write(ab + ba);
end.
""";
ValidateLexicalTokens(LexicalScanner.PascalParser(), pascalProgram, [
(LexicalTokenType.Keyword, "program"),
(LexicalTokenType.Identifier, "main"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "var"),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Delimiter, ","),
(LexicalTokenType.Identifier, "ba"),
(LexicalTokenType.Delimiter, ":"),
(LexicalTokenType.Keyword, "integer"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "begin"),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Operator, ":="),
(LexicalTokenType.ConstInteger, "3"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Identifier, "ba"),
(LexicalTokenType.Operator, ":="),
(LexicalTokenType.ConstInteger, "5"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Operator, ":="),
(LexicalTokenType.ConstInteger, "5"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Identifier, "write"),
(LexicalTokenType.Delimiter, "("),
(LexicalTokenType.Identifier, "ab"),
(LexicalTokenType.Operator, "+"),
(LexicalTokenType.Identifier, "ba"),
(LexicalTokenType.Delimiter, ")"),
(LexicalTokenType.Delimiter, ";"),
(LexicalTokenType.Keyword, "end"),
(LexicalTokenType.Delimiter, ".")
]);
}
[Theory]
[InlineData("""
program exFunction;
var
a, b, ret : integer;
begin
a := 100;
b := 200;
{ calling a function to get max valued }
ret := a - b;
end.
""", 29)]
[InlineData("""
{
This is a block comment that does closed.
}
program CommentClosed;
""", 3)]
[InlineData("""
{}
program CommentClosed;
""", 3)]
public void LexicalParserTest(string input, int count)
{
LexicalScanner scanner = new();
List<LexicalToken> tokens = scanner.Tokenize(new StringReadState(input)).ToList();
Assert.Equal(count, tokens.Count);
}
}

177
CanonSharp.Tests/ScannerTest/LexicalTokenParserTest.cs Normal file
View File

@@ -0,0 +1,177 @@
using CanonSharp.Combinator;
using CanonSharp.Combinator.Abstractions;
using CanonSharp.Combinator.Extensions;
using CanonSharp.Common.Scanner;
using CanonSharp.Tests.Utils;
namespace CanonSharp.Tests.ScannerTest;
public class LexicalTokenParserTest : LexicalTestBase
{
[Theory]
[InlineData("program")]
[InlineData("const")]
[InlineData("var")]
[InlineData("procedure")]
[InlineData("function")]
[InlineData("begin")]
[InlineData("end")]
[InlineData("array")]
[InlineData("of")]
[InlineData("if")]
[InlineData("then")]
[InlineData("else")]
[InlineData("for")]
[InlineData("to")]
[InlineData("do")]
[InlineData("true")]
[InlineData("false")]
[InlineData("while")]
public void KeywordParserTest(string literalValue)
{
Parser<char, LexicalToken> keyword = LexicalScanner.KeywordParser();
ValidateSuccessfulParser(keyword, LexicalTokenType.Keyword, literalValue, literalValue);
}
[Theory]
[InlineData("andOne")]
[InlineData("program1")]
[InlineData("todo")]
public void FailedKeywordParserTest(string input)
{
Parser<char, LexicalToken> keyword = LexicalScanner.KeywordParser();
ValidateFailedParser(keyword, input);
}
[Theory]
[InlineData(",")]
[InlineData(".")]
[InlineData(";")]
[InlineData(":")]
[InlineData("(")]
[InlineData(")")]
[InlineData("[")]
[InlineData("]")]
[InlineData("..")]
public void DelimiterParserTest(string literalValue)
{
Parser<char, LexicalToken> delimiter = LexicalScanner.DelimiterParser();
ValidateSuccessfulParser(delimiter, LexicalTokenType.Delimiter, literalValue, literalValue);
}
[Theory]
[InlineData(":=")]
public void FailedDelimiterParserTest(string input)
{
Parser<char, LexicalToken> delimiter = LexicalScanner.DelimiterParser();
ValidateFailedParser(delimiter, input);
}
[Theory]
[InlineData("=")]
[InlineData("!=")]
[InlineData(">")]
[InlineData(">=")]
[InlineData("<")]
[InlineData("<=")]
[InlineData("+")]
[InlineData("-")]
[InlineData("*")]
[InlineData("/")]
[InlineData(":=")]
public void OperatorParserTest(string literalValue)
{
Parser<char, LexicalToken> operatorParser = LexicalScanner.OperatorParser();
ValidateSuccessfulParser(operatorParser, LexicalTokenType.Operator, literalValue, literalValue);
}
[Theory]
[InlineData("identifier")]
[InlineData("_identifier")]
[InlineData("identifier123")]
[InlineData("identifier_with_underscore")]
[InlineData("CamelCase")]
[InlineData("andand")]
public void IdentifierParserTest(string literalValue)
{
Parser<char, LexicalToken> identifier = LexicalScanner.IdentifierParser();
ValidateSuccessfulParser(identifier, LexicalTokenType.Identifier, literalValue, literalValue);
}
[Theory]
[InlineData(123, "123")]
[InlineData(0, "0")]
public void ConstIntegerTest(int value, string input)
{
StringReadState state = new(input);
ParseResult<char, LexicalToken> result = LexicalScanner.ConstIntegerParser().Parse(state);
Assert.Equal(LexicalTokenType.ConstInteger, result.Value.TokenType);
Assert.Equal(value, int.Parse(result.Value.LiteralValue));
}
[Theory]
[InlineData(123.456, "123.456")]
[InlineData(0, "0.0")]
public void ConstFloatTest(double value, string input)
{
StringReadState state = new(input);
ParseResult<char, LexicalToken> result = LexicalScanner.ConstFloatParser().Parse(state);
Assert.Equal(LexicalTokenType.ConstFloat, result.Value.TokenType);
Assert.Equal(value, double.Parse(result.Value.LiteralValue));
}
[Theory]
[InlineData('a', "'a'")]
[InlineData('Z', "'Z'")]
public void CharTest(char value, string input)
{
StringReadState state = new(input);
ParseResult<char, LexicalToken> result = LexicalScanner.CharParser().Parse(state);
Assert.Equal(LexicalTokenType.Character, result.Value.TokenType);
Assert.Equal(value, char.Parse(result.Value.LiteralValue));
}
[Theory]
[InlineData("hello, world!", "'hello, world!'")]
public void StringTest(string value, string input)
{
StringReadState state = new(input);
ParseResult<char, LexicalToken> result = LexicalScanner.CharParser().Parse(state);
Assert.Equal(LexicalTokenType.String, result.Value.TokenType);
Assert.Equal(value, result.Value.LiteralValue);
}
[Theory]
[InlineData("{comment}")]
[InlineData("{}")]
public void CommentTest(string input)
{
StringReadState state = new(input);
ParseResult<char, Unit> result = LexicalScanner.CommentParser().Parse(state);
Assert.Equal(Unit.Instance, result.Value);
}
[Theory]
[InlineData(" {comment} program")]
[InlineData("""
{comment}
{comment}
{}{}{}{}
program
""")]
public void JunkTest(string input)
{
StringReadState state = new(input);
Parser<char, LexicalToken> parser = LexicalScanner.JunkParser().SkipTill(LexicalScanner.KeywordParser());
ParseResult<char, LexicalToken> result = parser.Parse(state);
Assert.Equal(LexicalTokenType.Keyword, result.Value.TokenType);
Assert.Equal("program", result.Value.LiteralValue);
}
}