fix: 区分字符和字符串 (#74)

Co-authored-by: Huaps <1183155719@qq.com>
Reviewed-on: PostGuard/Canon#74

Canon.Core/Enums/SemanticEnums.cs

@@ -8,6 +8,7 @@ public enum SemanticTokenType
     Delimiter,
     Identifier,
     Character,
+    String,
     Empty,
     /// <summary>
     /// 语法分析中的栈底符号

Canon.Core/LexicalParser/LexemeFactory.cs

@@ -16,6 +16,13 @@ public static class LexemeFactory
                 };
                 token = characterSemanticToken;
                 break;
+            case SemanticTokenType.String:
+                StringSemanticToken stringSemanticToken = new()
+                {
+                    LinePos = line, CharacterPos = chPos, LiteralValue = literal,
+                };
+                token = stringSemanticToken;
+                break;
             case SemanticTokenType.Identifier:
                 IdentifierSemanticToken identifierSemanticToken = new()
                 {

Canon.Core/LexicalParser/Lexer.cs

@@ -80,6 +80,7 @@ public class Lexer : ILexer
         }
 
         _tokens.Add(SemanticToken.End);
+
         return _tokens;
     }
 
@@ -447,7 +448,6 @@ public class Lexer : ILexer
                 }
                 break;
             case '\'':
-            case '\"':
                 {
                     // 重置_token，准备收集字符串内容
                     ResetTokenBuilder();
@@ -464,8 +464,18 @@ public class Lexer : ILexer
                         }
                     }
 
+                    string currentString = GetCurrentTokenString();
+                    if (currentString.Length > 1)
+                    {
+                        _semanticToken = LexemeFactory.MakeToken(SemanticTokenType.String,
+                            currentString, _line, _chPos);
+                    }
+                    else
+                    {
                         _semanticToken = LexemeFactory.MakeToken(SemanticTokenType.Character,
-                        GetCurrentTokenString(), _line, _chPos);
+                            currentString, _line, _chPos);
+                    }
+
 
                     ResetTokenBuilder();
 

Canon.Core/LexicalParser/SemanticToken.cs

@@ -104,6 +104,32 @@ public abstract class SemanticToken : IEquatable<SemanticToken>
 public class CharacterSemanticToken : SemanticToken
 {
     public override SemanticTokenType TokenType => SemanticTokenType.Character;
+
+    /// <summary>
+    /// 获得令牌代表的字符
+    /// </summary>
+    /// <returns>字符</returns>
+    public char ParseAsCharacter()
+    {
+        return char.Parse(LiteralValue);
+    }
+}
+
+/// <summary>
+/// 字符串类型记号
+/// </summary>
+public class StringSemanticToken : SemanticToken
+{
+    public override SemanticTokenType TokenType => SemanticTokenType.String;
+
+    /// <summary>
+    /// 获得令牌代表的字符串
+    /// </summary>
+    /// <returns>字符串</returns>
+    public string ParseAsString()
+    {
+        return LiteralValue;
+    }
 }
 
 /// <summary>

Canon.Tests/GrammarParserTests/PascalGrammarTests.cs

@@ -39,7 +39,7 @@ public class PascalGrammarTests
     {
         const string program = """
                                program exFunction;
-                               const str = 'result is : ';
+                               const str = 'a';
                                var a, b : Integer;
                                begin
                                writeln( str, ret );

Canon.Tests/LexicalParserTests/CharacterTypeTests.cs

@@ -7,33 +7,33 @@ using Canon.Tests.Utils;
 
 namespace Canon.Tests.LexicalParserTests
 {
-    public class CharacterTypeTests
+    public class CharacterTypeTests(ITestOutputHelper testOutputHelper)
     {
-        private readonly ITestOutputHelper _testOutputHelper;
         private readonly ILexer _lexer = new Lexer();
-        public CharacterTypeTests(ITestOutputHelper testOutputHelper)
-        {
-            _testOutputHelper = testOutputHelper;
-        }
 
         [Theory]
-        [InlineData("'a'", "a")]
-        [InlineData("'Hello, World!'", "Hello, World!")]
-
-        public void TestCharacterType(string input, string? expectedResult)
+        [InlineData("'a'", 'a')]
+        [InlineData("'+'", '+')]
+        public void TestCharacterType(string input, char expectedResult)
         {
             IEnumerable<SemanticToken> tokensEnumerable = _lexer.Tokenize(new StringSourceReader(input));
             List<SemanticToken> tokens = tokensEnumerable.ToList();
-            if (expectedResult == null)
-            {
-                Assert.Throws<LexemeException>(() => tokens);
-            }
-            else
-            {
-                _testOutputHelper.WriteLine(tokens[0].LiteralValue);
+
+            testOutputHelper.WriteLine(tokens[0].LiteralValue);
             Assert.Equal(SemanticTokenType.Character, tokens[0].TokenType);
-                Assert.Equal(expectedResult, tokens[0].LiteralValue);
+            Assert.Equal(expectedResult, tokens[0].Convert<CharacterSemanticToken>().ParseAsCharacter());
         }
+
+        [Theory]
+        [InlineData("'Hello, world'", "Hello, world")]
+        [InlineData("'asdfasdf'", "asdfasdf")]
+        public void StringTypeTest(string input, string expect)
+        {
+            IEnumerable<SemanticToken> tokens = _lexer.Tokenize(new StringSourceReader(input));
+            SemanticToken token = tokens.First();
+
+            Assert.Equal(SemanticTokenType.String, token.TokenType);
+            Assert.Equal(expect, token.Convert<StringSemanticToken>().ParseAsString());
         }
 
         [Theory]
@@ -42,11 +42,11 @@ namespace Canon.Tests.LexicalParserTests
         [InlineData("'This", 1, 5, LexemeErrorType.UnclosedStringLiteral)]
         [InlineData("x @", 1, 3, LexemeErrorType.UnknownCharacterOrString)]
         //[InlineData("\"x\'", 1, 3, LexemeException.LexemeErrorType.UnclosedStringLiteral)]
-        public void TestParseCharacterError(string input,  uint expectedLine, uint expectedCharPosition, LexemeErrorType expectedErrorType)
+        public void TestParseCharacterError(string input, uint expectedLine, uint expectedCharPosition,
+            LexemeErrorType expectedErrorType)
         {
-
             var ex = Assert.Throws<LexemeException>(() => _lexer.Tokenize(new StringSourceReader(input)).ToList());
-            _testOutputHelper.WriteLine(ex.ToString());
+            testOutputHelper.WriteLine(ex.ToString());
             Assert.Equal(expectedErrorType, ex.ErrorType);
             Assert.Equal(expectedLine, ex.Line);
             Assert.Equal(expectedCharPosition, ex.CharPosition);

Canon.Tests/LexicalParserTests/LexicalFileTests.cs

@@ -17,9 +17,9 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
         string pascalProgram = """
                                program HelloWorld;
                                var
-                               message: string;
+                               message: char;
                                begin
-                               message := 'hello, world!';
+                               message := 'h';
                                writeln(message);
                                end.
                                """;
@@ -32,7 +32,7 @@ public class LexicalFileTests(ITestOutputHelper testOutputHelper)
             SemanticTokenType.Keyword,
             SemanticTokenType.Identifier,
             SemanticTokenType.Delimiter,
-            SemanticTokenType.Identifier,
+            SemanticTokenType.Keyword,
             SemanticTokenType.Delimiter,
             SemanticTokenType.Keyword,
             SemanticTokenType.Identifier,