feat: Parser Combinator库和词法分析器 (#2)

Reviewed-on: https://git.bupt-hpc.cn/jackfiled/CanonSharp/pulls/2
Co-authored-by: jackfiled <xcrenchangjun@outlook.com>
Co-committed-by: jackfiled <xcrenchangjun@outlook.com>

CanonSharp.Tests/CombinatorsTests/BasicParsersTests.cs

@@ -0,0 +1,84 @@
+using CanonSharp.Combinator;
+using CanonSharp.Combinator.Abstractions;
+using CanonSharp.Combinator.Extensions;
+using CanonSharp.Tests.Utils;
+using static CanonSharp.Combinator.ParserBuilder;
+
+namespace CanonSharp.Tests.CombinatorsTests;
+
+public class BasicParsersTests : ParserTestsBase
+{
+    [Fact]
+    public void AlternativeTest()
+    {
+        Parser<char, char> parser = Token('a') | Token('b');
+
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+        ValidateFailedResult(parser, "cde");
+
+        parser = Token('a').Alternative(_ => Token('b'));
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+        ValidateFailedResult(parser, "cde");
+    }
+
+    [Fact]
+    public void BindTest()
+    {
+        Parser<char, char> parser = Token('a').Bind(_ => Token('b')).Bind(_ => Token('c'));
+        ValidateSuccessfulResult(parser, 'c', "abc");
+        ValidateFailedResult(parser, "acd");
+
+        ValidateFailedResult(parser, "ab");
+    }
+
+    [Fact]
+    public void MapTest()
+    {
+        Parser<char, string> parser = Token('a').Map(c => $"{c}");
+        ValidateSuccessfulResult(parser, "a", "abc");
+
+        parser = Token('a').Map("test");
+        ValidateSuccessfulResult(parser, "test", "abc");
+    }
+
+    [Fact]
+    public void NextTest()
+    {
+        Parser<char, char> parser = Token('a').Next(_ => Token('a'), _ => Token('b'));
+        ValidateSuccessfulResult(parser, 'a', "aaa");
+        ValidateSuccessfulResult(parser, 'b', "bbb");
+
+        parser = Token('a').Next(_ => Token('a'), _ => 'b');
+        ValidateSuccessfulResult(parser, 'b', "bbb");
+
+        parser = Token('a').Next(_ => Pure<char, char>('1'), '2');
+        ValidateSuccessfulResult(parser, '1', "aaa");
+        ValidateSuccessfulResult(parser, '2', "bbb");
+    }
+
+    [Fact]
+    public void NextTest2()
+    {
+        Parser<char, string> parser = Token('a').Next(_ => "123", _ => Pure<char, string>("456"));
+        ValidateSuccessfulResult(parser, "123", "aaa");
+        ValidateSuccessfulResult(parser, "456", "bbb");
+
+        parser = Token('a').Next(_ => "123", _ => "456");
+        ValidateSuccessfulResult(parser, "123", "aaa");
+        ValidateSuccessfulResult(parser, "456", "bbb");
+
+        parser = Token('a').Next(_ => "123", "456");
+        ValidateSuccessfulResult(parser, "123", "aaa");
+        ValidateSuccessfulResult(parser, "456", "bbb");
+    }
+
+    [Fact]
+    public void FixTest()
+    {
+        Parser<char, char> parser = Fix<char, Unit>(self => Token('a').Next(_ => self, Unit.Instance))
+            .Bind(_ => Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "aaaaab");
+    }
+}

CanonSharp.Tests/CombinatorsTests/CombinatorParserTests.cs

@@ -0,0 +1,176 @@
+using CanonSharp.Combinator;
+using CanonSharp.Combinator.Abstractions;
+using CanonSharp.Combinator.Extensions;
+using CanonSharp.Tests.Utils;
+using static CanonSharp.Combinator.ParserBuilder;
+
+namespace CanonSharp.Tests.CombinatorsTests;
+
+public class CombinatorParserTests : ParserTestsBase
+{
+    [Fact]
+    public void ChoiceTest()
+    {
+        Parser<char, char> parser = Choice(Token('a'), Token('b'), Token('c'));
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+        ValidateSuccessfulResult(parser, 'c', "cde");
+
+        parser = Choice([Token('a'), Token('b'), Token('c')]);
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+        ValidateSuccessfulResult(parser, 'c', "cde");
+    }
+
+    [Fact]
+    public void SequenceTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Sequence(Token('a'), Token('b'), Token('c'));
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "abc");
+
+        parser = Sequence([Token('a'), Token('b'), Token('c')]);
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "abc");
+    }
+
+    [Fact]
+    public void LeftRightTest()
+    {
+        Parser<char, char> parser = Token('a').Left(Token('b'));
+        ValidateSuccessfulResult(parser, 'a', "ab");
+
+        parser = Token('a').Right(Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "ab");
+    }
+
+    [Fact]
+    public void ManyTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Token('a').Many();
+        ValidateSuccessfulResult(parser, [], "bbb");
+        ValidateSuccessfulResult(parser, ['a', 'a', 'a'], "aaa");
+
+        parser = Token('a').Many1();
+        ValidateSuccessfulResult(parser, ['a', 'a'], "aa");
+        ValidateFailedResult(parser, "bbb");
+    }
+
+    [Fact]
+    public void SkipManyTest()
+    {
+        Parser<char, char> parser = Token('a').SkipMany().Right(Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "aaaab");
+        ValidateSuccessfulResult(parser, 'b', "bbbb");
+
+        parser = Token('a').SkipMany1().Right(Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "aaaaaab");
+        ValidateFailedResult(parser, "bb");
+    }
+
+    [Fact]
+    public void ChainTest()
+    {
+        // 等效于Many1
+        // 但是不返回中间结果
+        Parser<char, char> parser = Token('a').Chain(Token);
+        ValidateSuccessfulResult(parser, 'a', "aa");
+        ValidateFailedResult(parser, "bb");
+
+        parser = Token('_').Chain(x => x == '_' ? Satisfy<char>(char.IsLetter) : Satisfy<char>(char.IsDigit));
+        ValidateSuccessfulResult(parser, '1', "_a1");
+        ValidateSuccessfulResult(parser, '_', "_123");
+    }
+
+    [Fact]
+    public void ManyTillTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Token('a').ManyTill(Token('b').LookAhead());
+        ValidateSuccessfulResult(parser, ['a', 'a', 'a'], "aaab");
+        ValidateSuccessfulResult(parser, [], "b");
+
+        parser = Token('a').Many1Till(Token('b').LookAhead());
+        ValidateSuccessfulResult(parser, ['a', 'a'], "aab");
+        ValidateFailedResult(parser, "bb");
+    }
+
+    [Fact]
+    public void SkipTillTest()
+    {
+        Parser<char, char> parser = Token('a').SkipTill(Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "aaab");
+        ValidateSuccessfulResult(parser, 'b', "b");
+
+        parser = Token('a').Skip1Till(Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "aaab");
+        ValidateFailedResult(parser, "b");
+    }
+
+    [Fact]
+    public void TakeTillTest()
+    {
+        Parser<char, IEnumerable<char>> parser = TakeTill(Token('b').LookAhead());
+        ValidateSuccessfulResult(parser, ['a', 'a'], "aab");
+        ValidateSuccessfulResult(parser, [], "b");
+
+        parser = Take1Till(Token('b').LookAhead());
+        ValidateSuccessfulResult(parser, ['a', 'a'], "aab");
+        ValidateFailedResult(parser, "b");
+    }
+
+    [Fact]
+    public void MatchTest()
+    {
+        Parser<char, char> parser = Token('b').Match();
+        ValidateSuccessfulResult(parser, 'b', "asdfasdfasdfasdfb");
+        ValidateSuccessfulResult(parser, 'b', "b");
+    }
+
+    [Fact]
+    public void QuoteTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Any<char>().Quote(Token('['), Token(']'));
+        ValidateSuccessfulResult(parser, ['1', '2', '3'], "[123]");
+
+        parser = Any<char>().Quote(Token('\''));
+        ValidateSuccessfulResult(parser, ['1', '2', '3'], "'123'");
+    }
+
+    [Fact]
+    public void SeparatedByTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Token('a').SeparatedBy(Token(','));
+        ValidateSuccessfulResult(parser, ['a', 'a', 'a'], "a,a,a");
+        ValidateSuccessfulResult(parser, ['a'], "a");
+        ValidateSuccessfulResult(parser, [], "");
+
+        parser = Token('a').SeparatedBy1(Token(','));
+        ValidateSuccessfulResult(parser, ['a', 'a', 'a'], "a,a,a");
+        ValidateSuccessfulResult(parser, ['a'], "a");
+        ValidateFailedResult(parser, "");
+    }
+
+    [Fact]
+    public void EndByTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Satisfy<char>(char.IsLetter).EndBy(Token('.'));
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "abc.");
+        ValidateSuccessfulResult(parser, [], ".");
+
+        parser = Satisfy<char>(char.IsLetter).EndBy1(Token('.'));
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "abc.");
+        ValidateFailedResult(parser, ".");
+    }
+
+    [Fact]
+    public void SeparatedOrEndByTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Satisfy<char>(char.IsLetter).SeparatedOrEndBy1(Token(','));
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "a,b,c,");
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "a,b,c");
+        ValidateFailedResult(parser, "");
+
+        parser = Satisfy<char>(char.IsLetter).SeparatedOrEndBy(Token(','));
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "a,b,c,");
+        ValidateSuccessfulResult(parser, ['a', 'b', 'c'], "a,b,c");
+        ValidateSuccessfulResult(parser, [], "");
+    }
+}

CanonSharp.Tests/CombinatorsTests/LinqTests.cs

@@ -0,0 +1,29 @@
+using CanonSharp.Combinator.Abstractions;
+using CanonSharp.Combinator.Extensions;
+using static CanonSharp.Combinator.Text.TextParserBuilder;
+using CanonSharp.Tests.Utils;
+
+namespace CanonSharp.Tests.CombinatorsTests;
+
+public class LinqTests : ParserTestsBase
+{
+    [Fact]
+    public void SelectTest1()
+    {
+        Parser<char, string> parser = from token in Char('a')
+            select token.ToString();
+        ValidateSuccessfulResult(parser, "a", "a");
+        ValidateFailedResult(parser, "b");
+    }
+
+    [Fact]
+    public void SelectManyTest1()
+    {
+        Parser<char, int> parser = from _1 in Char('a')
+            from _2 in Char('b')
+            from _3 in Char('c')
+            select 123;
+        ValidateSuccessfulResult(parser, 123, "abc");
+        ValidateFailedResult(parser, "asd");
+    }
+}

CanonSharp.Tests/CombinatorsTests/ModifierParserTests.cs

@@ -0,0 +1,51 @@
+using CanonSharp.Combinator;
+using CanonSharp.Combinator.Abstractions;
+using CanonSharp.Combinator.Extensions;
+using CanonSharp.Tests.Utils;
+using static CanonSharp.Combinator.ParserBuilder;
+using static CanonSharp.Combinator.Text.TextParserBuilder;
+
+namespace CanonSharp.Tests.CombinatorsTests;
+
+public class ModifierParserTests : ParserTestsBase
+{
+    [Fact]
+    public void DoTest()
+    {
+        Parser<char, char> parser = Token('a').Do(x => Assert.Equal('a', x)).Do(x => Assert.Equal('a', x),
+            failedResult => Assert.ThrowsAny<ParseException>(() => failedResult.Value));
+
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateFailedResult(parser, "bcd");
+    }
+
+    [Fact]
+    public void LookAheadTest()
+    {
+        Parser<char, char> parser = Token('a').LookAhead().Next(_ => Token('a'), _ => Token('b'));
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+    }
+
+    [Fact]
+    public void NotTest()
+    {
+        Parser<char, char> parser = Token('a').Not('b');
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+
+        parser = Token('a').Not().Bind(_ => Token('b'));
+        ValidateSuccessfulResult(parser, 'b', "bcd");
+    }
+
+    [Fact]
+    public void TryTest()
+    {
+        Parser<char, string> parser = String("abc").Try("cde");
+        ValidateSuccessfulResult(parser, "abc", "abc");
+        ValidateSuccessfulResult(parser, "cde", "cde");
+
+        parser = String("abc").Try(_ => "cde");
+        ValidateSuccessfulResult(parser, "abc", "abc");
+        ValidateSuccessfulResult(parser, "cde", "cde");
+    }
+}

CanonSharp.Tests/CombinatorsTests/PrimitiveParserTests.cs

@@ -0,0 +1,84 @@
+using CanonSharp.Combinator;
+using CanonSharp.Combinator.Abstractions;
+using CanonSharp.Combinator.Extensions;
+using CanonSharp.Tests.Utils;
+using static CanonSharp.Combinator.ParserBuilder;
+
+namespace CanonSharp.Tests.CombinatorsTests;
+
+public class PrimitiveParserTests : ParserTestsBase
+{
+    [Fact]
+    public void PureTest()
+    {
+        Parser<char, char> parser = Pure<char, char>('a');
+        ValidateSuccessfulResult(parser, 'a', "abc");
+
+        parser = Pure<char, char>(_ => 'a');
+        ValidateSuccessfulResult(parser, 'a', "abc");
+    }
+
+    [Fact]
+    public void NullTest()
+    {
+        Parser<char, Unit> parser = Null<char>();
+        ValidateSuccessfulResult(parser, Unit.Instance, "abc");
+    }
+
+    [Fact]
+    public void FailTest()
+    {
+        Parser<char, char> parser = Fail<char, char>();
+        ValidateFailedResult(parser, "abc");
+
+        parser = Fail<char, char>("Failed message");
+        FailedResult<char, char> result = ValidateFailedResult(parser, "abc");
+        Assert.Equal("Failed message", result.Message);
+
+        parser = Fail<char, char>(x => $"{x}");
+        result = ValidateFailedResult(parser, "abc");
+        Assert.Equal("a<0x61>", result.Message);
+
+        parser = Fail<char, char>(new InvalidOperationException());
+        result = ValidateFailedResult(parser, "abc");
+        Assert.IsType<InvalidOperationException>(result.Exception.InnerException);
+    }
+
+    [Fact]
+    public void SatisfyTest()
+    {
+        Parser<char, char> parser = Satisfy<char>(char.IsLetter);
+        ValidateSuccessfulResult(parser, 'a', "abc");
+        ValidateFailedResult(parser, "123");
+    }
+
+    [Fact]
+    public void AnyTest()
+    {
+        Parser<char, char> parser = Any<char>();
+        ValidateSuccessfulResult(parser, '1', "123");
+    }
+
+    [Fact]
+    public void TokenTest()
+    {
+        Parser<char, char> parser = Token('a');
+        ValidateSuccessfulResult(parser, 'a', "abc");
+    }
+
+    [Fact]
+    public void TakeTest()
+    {
+        Parser<char, IEnumerable<char>> parser = Take<char>(5);
+        ValidateSuccessfulResult(parser, ['h', 'e', 'l', 'l', 'o'], "hello");
+        ValidateFailedResult(parser, "abc");
+    }
+
+    [Fact]
+    public void SkipTest()
+    {
+        Parser<char, char> parser = Skip<char>(5).Bind(_ => Token(','));
+        ValidateSuccessfulResult(parser, ',', "hello,world.");
+        ValidateFailedResult(parser, "abc");
+    }
+}