add: tests for zero-parser.

tests/file_lexical_parser_tests.rs

@@ -0,0 +1,105 @@
+mod tokenizer;
+
+use crate::tokenizer::{nom_lexical_parser, zero_lexical_parser};
+use anyhow::anyhow;
+use std::fs::File;
+use std::io::{BufReader, Read};
+use std::path::PathBuf;
+use std::{env, fs};
+use zero_parser::parser::ParserContext;
+
+pub fn ensure_sysy_sources() -> anyhow::Result<Vec<PathBuf>> {
+    let current_dir = env::current_dir()?;
+
+    for entry in fs::read_dir(&current_dir)? {
+        let entry = entry?;
+
+        if entry.file_name() == "sysy_sets" {
+            let set_path = entry.path();
+
+            return Ok(fs::read_dir(&set_path)?
+                .filter_map(|f| {
+                    f.ok().and_then(|f| {
+                        f.file_name()
+                            .into_string()
+                            .ok()
+                            .and_then(|file_name| file_name.ends_with(".sy").then_some(f.path()))
+                    })
+                })
+                .collect());
+        }
+    }
+
+    Err(anyhow!(
+        "Failed to find `sys_sets` directory in current dir {:?}.",
+        current_dir.file_name().unwrap()
+    ))
+}
+
+#[derive(Debug)]
+pub struct SourceFile {
+    pub filename: String,
+    pub content: String,
+}
+
+impl SourceFile {
+    pub fn new(path: &PathBuf) -> anyhow::Result<Self> {
+        let file = File::open(path)?;
+        let mut reader = BufReader::new(file);
+
+        let mut content = String::new();
+        reader.read_to_string(&mut content)?;
+
+        Ok(Self {
+            filename: path
+                .file_name()
+                .and_then(|x| x.to_str())
+                .ok_or(anyhow!("Failed to get filename from path."))?
+                .to_owned(),
+            content,
+        })
+    }
+}
+
+#[test]
+fn file_lexical_parser_test() -> anyhow::Result<()> {
+    let sysy_set = ensure_sysy_sources()?;
+    let source_files = sysy_set
+        .into_iter()
+        .map(|p| SourceFile::new(&p))
+        .collect::<Result<Vec<SourceFile>, anyhow::Error>>()?;
+
+    for source_file in source_files {
+        println!("Start to test file '{}'.", source_file.filename);
+
+        let (_, nom_tokens) =
+            nom_lexical_parser(source_file.content.as_str()).or_else(|e| Err(e.to_owned()))?;
+
+        let context = ParserContext::new_with_str(source_file.content.as_str(), ());
+        let borrowed_context = context.borrow();
+        let (_, zero_tokens) = zero_lexical_parser(context.clone(), borrowed_context.input_slice())
+            .or_else(|e| Err(anyhow!("{}", e.1)))?;
+
+        assert_eq!(nom_tokens.len(), zero_tokens.len());
+
+        for (except, actual) in nom_tokens.iter().zip(zero_tokens.iter()) {
+            assert_eq!(
+                except,
+                actual,
+                "The literal value of actual token is {}.",
+                actual
+                    .literal_value
+                    .iter()
+                    .map(|x| x.clone())
+                    .collect::<String>()
+            );
+        }
+        println!(
+            "Succeed to test file '{}', testing {} tokens.",
+            source_file.filename,
+            zero_tokens.len()
+        );
+    }
+
+    Ok(())
+}

tests/lexical_parser_tests.rs

@@ -0,0 +1,203 @@
+use crate::tokenizer::LexicalTokenType::{ConstInteger, Delimiter, Identifier, Keyword, Operator};
+use crate::tokenizer::{zero_lexical_parser, LexicalTokenType};
+use zero_parser::parser::ParserContext;
+
+mod tokenizer;
+
+fn validate_tokens(input: &'static str, tokens: Vec<LexicalTokenType>) {
+    let context = ParserContext::new_with_str(input, ());
+    let borrowed_context = context.borrow();
+
+    let (_, actual_tokens) =
+        zero_lexical_parser(context.clone(), borrowed_context.input_slice()).unwrap();
+
+    dbg!(&tokens, &actual_tokens);
+    assert_eq!(tokens.len(), actual_tokens.len());
+
+    for (actual, except) in actual_tokens.iter().zip(tokens.iter()) {
+        assert_eq!(
+            &actual.token_type,
+            except,
+            "The literal value of actual token is {}.",
+            actual
+                .literal_value
+                .iter()
+                .map(|x| x.clone())
+                .collect::<String>()
+        );
+    }
+}
+
+#[test]
+fn main_test() {
+    validate_tokens(
+        "int main() { return 0; }",
+        vec![
+            Keyword,
+            Identifier,
+            Delimiter,
+            Delimiter,
+            Delimiter,
+            Keyword,
+            ConstInteger(0),
+            Delimiter,
+            Delimiter,
+        ],
+    );
+}
+
+#[test]
+fn number_test() {
+    validate_tokens("123", vec![ConstInteger(123)])
+}
+
+#[test]
+fn hexadecimal_test() {
+    validate_tokens(
+        "// test hexadecimal define
+int main(){
+    int a;
+    a = 0xf;
+    return a;
+}",
+        vec![
+            Keyword,
+            Identifier,
+            Delimiter,
+            Delimiter,
+            Delimiter,
+            Keyword,
+            Identifier,
+            Delimiter,
+            Identifier,
+            Operator,
+            ConstInteger(15),
+            Delimiter,
+            Keyword,
+            Identifier,
+            Delimiter,
+            Delimiter,
+        ],
+    );
+}
+
+#[test]
+fn while_and_if_test() {
+    validate_tokens(
+        "
+    // test while-if
+    int whileIf() {
+        int a;
+        a = 0;
+        int b;
+        b = 0;
+        while (a < 100) {
+            if (a == 5) {
+                b = 25;
+            }
+            else if (a == 10) {
+                b = 42;
+            }
+            else {
+                b = a * 2;
+            }
+            a = a + 1;
+        }
+        return (b);
+    }",
+        vec![
+            // int whileIf() {
+            Keyword,
+            Identifier,
+            Delimiter,
+            Delimiter,
+            Delimiter,
+            // int a;
+            Keyword,
+            Identifier,
+            Delimiter,
+            // a = 0;
+            Identifier,
+            Operator,
+            ConstInteger(0),
+            Delimiter,
+            // int b;
+            Keyword,
+            Identifier,
+            Delimiter,
+            // b = 0;
+            Identifier,
+            Operator,
+            ConstInteger(0),
+            Delimiter,
+            // while (a < 100) {
+            Keyword,
+            Delimiter,
+            Identifier,
+            Operator,
+            ConstInteger(100),
+            Delimiter,
+            Delimiter,
+            // if (a == 5) {
+            Keyword,
+            Delimiter,
+            Identifier,
+            Operator,
+            ConstInteger(5),
+            Delimiter,
+            Delimiter,
+            // b = 25;
+            Identifier,
+            Operator,
+            ConstInteger(25),
+            Delimiter,
+            // }
+            Delimiter,
+            // else if (a == 10) {
+            Keyword,
+            Keyword,
+            Delimiter,
+            Identifier,
+            Operator,
+            ConstInteger(10),
+            Delimiter,
+            Delimiter,
+            // b = 42;
+            Identifier,
+            Operator,
+            ConstInteger(42),
+            Delimiter,
+            // }
+            Delimiter,
+            // else {
+            Keyword,
+            Delimiter,
+            // b = a * 2;
+            Identifier,
+            Operator,
+            Identifier,
+            Operator,
+            ConstInteger(2),
+            Delimiter,
+            // }
+            Delimiter,
+            // a = a + 1;
+            Identifier,
+            Operator,
+            Identifier,
+            Operator,
+            ConstInteger(1),
+            Delimiter,
+            // }
+            Delimiter,
+            // return (b);
+            Keyword,
+            Delimiter,
+            Identifier,
+            Delimiter,
+            Delimiter,
+            // }
+            Delimiter,
+        ],
+    );
+}

tests/tokenizer/mod.rs

@@ -0,0 +1,133 @@
+use nom::IResult;
+use std::cell::RefCell;
+use std::rc::Rc;
+use zero_parser::parser::{ParserContext, ParserResult};
+
+mod nom_parsers;
+pub mod zero_parsers;
+
+/// 词法令牌的类别
+#[derive(Debug, Copy, Clone)]
+pub enum LexicalTokenType {
+    Identifier,
+    /// 整型常数类型
+    /// 在词法分析阶段就得到常量的字面值
+    ConstInteger(u32),
+    /// 浮点常数类型
+    ConstFloat(f32),
+    Keyword,
+    Delimiter,
+    Operator,
+    LiteralString,
+}
+
+/// 词法令牌
+#[derive(PartialEq, Debug, Copy, Clone)]
+pub struct LexicalToken<'a> {
+    pub token_type: LexicalTokenType,
+    pub literal_value: &'a str,
+}
+
+#[derive(PartialEq, Debug, Copy, Clone)]
+pub struct NewLexicalToken<'a> {
+    pub token_type: LexicalTokenType,
+    pub literal_value: &'a [char],
+}
+
+/// 为词法令牌类型实现相等性判断
+/// 重写判断的原因是我们希望不同值类型的整型常量和浮点常量可以是相同的
+///
+/// 即满足如下的判断：
+/// ```
+/// use rustic_sysy::tokenizer::LexicalTokenType;
+///
+/// assert_eq!(LexicalTokenType::ConstInteger(0), LexicalTokenType::ConstInteger(2));
+/// assert_eq!(LexicalTokenType::ConstFloat(0f32), LexicalTokenType::ConstFloat(2f32));
+/// ```
+impl PartialEq for LexicalTokenType {
+    fn eq(&self, other: &Self) -> bool {
+        match self {
+            LexicalTokenType::ConstInteger(_) => match other {
+                LexicalTokenType::ConstInteger(_) => true,
+                _ => false,
+            },
+            LexicalTokenType::ConstFloat(_) => match other {
+                LexicalTokenType::ConstFloat(_) => true,
+                _ => false,
+            },
+            LexicalTokenType::LiteralString => match other {
+                LexicalTokenType::LiteralString => true,
+                _ => false,
+            },
+            LexicalTokenType::Keyword => match other {
+                LexicalTokenType::Keyword => true,
+                _ => false,
+            },
+            LexicalTokenType::Identifier => match other {
+                LexicalTokenType::Identifier => true,
+                _ => false,
+            },
+            LexicalTokenType::Delimiter => match other {
+                LexicalTokenType::Delimiter => true,
+                _ => false,
+            },
+            LexicalTokenType::Operator => match other {
+                LexicalTokenType::Operator => true,
+                _ => false,
+            },
+        }
+    }
+}
+
+impl<'a, 'b> PartialEq<NewLexicalToken<'a>> for LexicalToken<'b> {
+    fn eq(&self, other: &NewLexicalToken) -> bool {
+        self.token_type == other.token_type
+            && self.literal_value.chars().collect::<String>()
+                == other.literal_value.iter().collect::<String>()
+    }
+}
+
+pub fn nom_lexical_parser(mut input: &str) -> IResult<&str, Vec<LexicalToken>> {
+    let mut array = vec![];
+
+    while !input.is_empty() {
+        if let Ok((i, _)) = nom_parsers::junk_parser(input) {
+            if i.is_empty() {
+                break;
+            }
+
+            input = i;
+            continue;
+        }
+
+        let (i, token) = nom_parsers::combine_parser(input)?;
+        input = i;
+        array.push(token);
+    }
+
+    Ok((input, array))
+}
+
+pub fn zero_lexical_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    mut input: &[char],
+) -> ParserResult<char, Vec<NewLexicalToken>> {
+    let mut array = vec![];
+
+    while !input.is_empty() {
+        if let Ok((i, _)) = zero_parsers::junk_parser(context.clone(), input) {
+            if i.is_empty() {
+                break;
+            }
+
+            input = i;
+            continue;
+        }
+
+        let (i, token) = zero_parsers::combine_parser(context.clone(), input)?;
+        input = i;
+        array.push(token);
+    }
+
+    Ok((input, array))
+}

tests/tokenizer/nom_parsers.rs

@@ -0,0 +1,180 @@
+use crate::tokenizer::{LexicalToken, LexicalTokenType};
+use nom::branch::alt;
+use nom::bytes::complete::{tag, take_until};
+use nom::character::complete::{
+    alpha1, alphanumeric1, digit0, digit1, hex_digit0, multispace1, oct_digit0, one_of,
+};
+use nom::combinator::{map, not, peek, recognize, value};
+use nom::multi::many0_count;
+use nom::sequence::{delimited, pair, tuple};
+use nom::IResult;
+use std::str::FromStr;
+
+fn keyword_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(
+        recognize(tuple((
+            alt((
+                tag("break"),
+                tag("const"),
+                tag("continue"),
+                tag("else"),
+                tag("float"),
+                tag("if"),
+                tag("int"),
+                tag("return"),
+                tag("void"),
+                tag("while"),
+            )),
+            not(alt((peek(alphanumeric1), tag("_")))),
+        ))),
+        |x| LexicalToken {
+            token_type: LexicalTokenType::Keyword,
+            literal_value: x,
+        },
+    )(input)
+}
+
+fn delimiter_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(
+        alt((
+            tag(","),
+            tag(";"),
+            tag("("),
+            tag(")"),
+            tag("["),
+            tag("]"),
+            tag("{"),
+            tag("}"),
+        )),
+        |x| LexicalToken {
+            token_type: LexicalTokenType::Delimiter,
+            literal_value: x,
+        },
+    )(input)
+}
+
+fn operator_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(
+        alt((
+            tag(">="),
+            tag("<="),
+            tag("=="),
+            tag("!="),
+            tag("&&"),
+            tag("||"),
+            tag("="),
+            tag("+"),
+            tag("-"),
+            tag("!"),
+            tag("*"),
+            tag("/"),
+            tag("%"),
+            tag(">"),
+            tag("<"),
+        )),
+        |x| LexicalToken {
+            token_type: LexicalTokenType::Operator,
+            literal_value: x,
+        },
+    )(input)
+}
+
+fn identifier_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(
+        recognize(pair(
+            alt((alpha1, tag("_"))),
+            many0_count(alt((alphanumeric1, tag("_")))),
+        )),
+        |s| LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: s,
+        },
+    )(input)
+}
+
+fn decimal_integer_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(recognize(pair(one_of("123456789"), digit0)), |x| {
+        let number = u32::from_str_radix(x, 10).unwrap();
+
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(number),
+            literal_value: x,
+        }
+    })(input)
+}
+
+fn octal_integer_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(recognize(pair(tag("0"), oct_digit0)), |x| {
+        let number = u32::from_str_radix(x, 8).unwrap();
+
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(number),
+            literal_value: x,
+        }
+    })(input)
+}
+
+fn hexadecimal_integer_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(
+        recognize(pair(alt((tag("0x"), tag("0X"))), hex_digit0)),
+        |x: &str| {
+            let number = u32::from_str_radix(&x[2..], 16).unwrap();
+
+            LexicalToken {
+                token_type: LexicalTokenType::ConstInteger(number),
+                literal_value: x,
+            }
+        },
+    )(input)
+}
+
+fn integer_parser(input: &str) -> IResult<&str, LexicalToken> {
+    alt((
+        hexadecimal_integer_parser,
+        octal_integer_parser,
+        decimal_integer_parser,
+    ))(input)
+}
+
+fn float_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(recognize(tuple((digit1, tag("."), digit1))), |x| {
+        let number = f32::from_str(x).unwrap();
+
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(number),
+            literal_value: x,
+        }
+    })(input)
+}
+
+fn literal_string_parser(input: &str) -> IResult<&str, LexicalToken> {
+    map(delimited(tag("\""), take_until("\""), tag("\"")), |s| {
+        LexicalToken {
+            token_type: LexicalTokenType::LiteralString,
+            literal_value: s,
+        }
+    })(input)
+}
+
+fn comments_parser(input: &str) -> IResult<&str, ()> {
+    alt((
+        value((), tuple((tag("//"), take_until("\n"), tag("\n")))),
+        value((), tuple((tag("/*"), take_until("*/"), tag("*/")))),
+    ))(input)
+}
+
+pub fn junk_parser(input: &str) -> IResult<&str, ()> {
+    alt((value((), multispace1), comments_parser))(input)
+}
+
+pub fn combine_parser(input: &str) -> IResult<&str, LexicalToken> {
+    alt((
+        float_parser,
+        integer_parser,
+        literal_string_parser,
+        keyword_parser,
+        identifier_parser,
+        delimiter_parser,
+        operator_parser,
+    ))(input)
+}

tests/tokenizer/zero_parsers.rs

@@ -0,0 +1,266 @@
+use crate::tokenizer::{LexicalTokenType, NewLexicalToken};
+use nom::AsChar;
+use std::cell::RefCell;
+use std::rc::Rc;
+use std::str::FromStr;
+use zero_parser::combinators::{quote, take_till, tuple, ParserExt};
+use zero_parser::parser::{any, Parser, ParserContext, ParserResult};
+use zero_parser::text::{char_parser, one_of, string_parser};
+use zero_parser::{alternate, parser::satisfy};
+
+pub fn keyword_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        alternate!(
+            string_parser("break"),
+            string_parser("const"),
+            string_parser("continue"),
+            string_parser("else"),
+            string_parser("float"),
+            string_parser("if"),
+            string_parser("int"),
+            string_parser("return"),
+            string_parser("void"),
+            string_parser("while")
+        ),
+        alternate!(satisfy(|c: &char| c.is_alphanumeric()), char_parser('_'))
+            .look_ahead()
+            .reverse(()),
+    ))
+    .literal()
+    .map(|x| NewLexicalToken {
+        token_type: LexicalTokenType::Keyword,
+        literal_value: x,
+    })
+    .parse(context, input)
+}
+
+pub fn delimiter_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    alternate!(
+        char_parser(','),
+        char_parser(';'),
+        char_parser('('),
+        char_parser(')'),
+        char_parser('['),
+        char_parser(']'),
+        char_parser('{'),
+        char_parser('}')
+    )
+    .literal()
+    .map(|x| NewLexicalToken {
+        token_type: LexicalTokenType::Delimiter,
+        literal_value: x,
+    })
+    .parse(context, input)
+}
+
+pub fn operator_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    alternate!(
+        string_parser(">="),
+        string_parser("<="),
+        string_parser("=="),
+        string_parser("!="),
+        string_parser("&&"),
+        string_parser("||"),
+        string_parser("="),
+        string_parser("+"),
+        string_parser("-"),
+        string_parser("!"),
+        string_parser("*"),
+        string_parser("/"),
+        string_parser("%"),
+        string_parser(">"),
+        string_parser("<")
+    )
+    .literal()
+    .map(|x| NewLexicalToken {
+        token_type: LexicalTokenType::Operator,
+        literal_value: x,
+    })
+    .parse(context, input)
+}
+
+pub fn identifier_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        alternate!(satisfy(|c: &char| c.is_alphabetic()), char_parser('_')),
+        alternate!(satisfy(|c: &char| c.is_alphanumeric()), char_parser('_')).many(),
+    ))
+    .literal()
+    .map(|x| NewLexicalToken {
+        token_type: LexicalTokenType::Identifier,
+        literal_value: x,
+    })
+    .parse(context, input)
+}
+
+pub fn decimal_integer_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        one_of("123456789"),
+        satisfy(|c: &char| c.is_ascii_digit()).many(),
+    ))
+    .literal()
+    .map(|x| {
+        let word: String = x.iter().map(|x| x.clone()).collect();
+        let number = u32::from_str_radix(word.as_str(), 10).unwrap();
+
+        NewLexicalToken {
+            token_type: LexicalTokenType::ConstInteger(number),
+            literal_value: x,
+        }
+    })
+    .parse(context, input)
+}
+
+pub fn octal_integer_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        char_parser('0'),
+        satisfy(|c: &char| c.is_oct_digit()).many(),
+    ))
+    .literal()
+    .map(|x| {
+        let word: String = x.iter().collect();
+        let number = u32::from_str_radix(word.as_str(), 8).unwrap();
+
+        NewLexicalToken {
+            token_type: LexicalTokenType::ConstInteger(number),
+            literal_value: x,
+        }
+    })
+    .parse(context, input)
+}
+
+pub fn hexadecimal_integer_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        alternate!(string_parser("0x"), string_parser("0X")),
+        satisfy(|c: &char| c.is_hex_digit()).many(),
+    ))
+    .literal()
+    .map(|x| {
+        let word: String = (&x[2..]).iter().collect();
+        let number = u32::from_str_radix(word.as_str(), 16).unwrap();
+
+        NewLexicalToken {
+            token_type: LexicalTokenType::ConstInteger(number),
+            literal_value: x,
+        }
+    })
+    .parse(context, input)
+}
+
+pub fn integer_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    alternate!(
+        hexadecimal_integer_parser,
+        octal_integer_parser,
+        decimal_integer_parser
+    )
+    .parse(context, input)
+}
+
+pub fn float_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    tuple((
+        satisfy(|c: &char| c.is_ascii_digit()).many1(),
+        char_parser('.'),
+        satisfy(|c: &char| c.is_ascii_digit()).many1(),
+    ))
+    .literal()
+    .map(|x| {
+        let word: String = x.iter().map(|c| c.clone()).collect();
+        let number = f32::from_str(word.as_str()).unwrap();
+
+        NewLexicalToken {
+            token_type: LexicalTokenType::ConstFloat(number),
+            literal_value: x,
+        }
+    })
+    .parse(context, input)
+}
+
+pub fn literal_string_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    quote(char_parser('"'), any(), char_parser('"'))
+        .literal()
+        .map(|x| {
+            let length = x.len();
+            NewLexicalToken {
+                token_type: LexicalTokenType::LiteralString,
+                literal_value: &x[1..length],
+            }
+        })
+        .parse(context, input)
+}
+
+pub fn comments_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, ()> {
+    alternate!(
+        tuple((
+            string_parser("//"),
+            take_till(char_parser('\n')),
+            char_parser('\n')
+        ))
+        .map(|_| ()),
+        tuple((
+            string_parser("/*"),
+            take_till(string_parser("*/")),
+            string_parser("*/")
+        ))
+        .map(|_| ())
+    )
+    .parse(context, input)
+}
+
+pub fn junk_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, ()> {
+    alternate!(
+        comments_parser,
+        satisfy(|c: &char| c.is_whitespace()).many1().map(|_| ())
+    )
+    .parse(context, input)
+}
+
+pub fn combine_parser(
+    context: Rc<RefCell<ParserContext<char, ()>>>,
+    input: &[char],
+) -> ParserResult<char, NewLexicalToken> {
+    alternate!(
+        float_parser,
+        integer_parser,
+        literal_string_parser,
+        keyword_parser,
+        identifier_parser,
+        delimiter_parser,
+        operator_parser
+    )
+    .parse(context, input)
+}

tests/zero_parser_tests.rs

@@ -0,0 +1,368 @@
+use crate::tokenizer::zero_parsers::{
+    combine_parser, comments_parser, decimal_integer_parser, delimiter_parser, float_parser,
+    hexadecimal_integer_parser, identifier_parser, integer_parser, keyword_parser,
+    literal_string_parser, octal_integer_parser, operator_parser,
+};
+use crate::tokenizer::{LexicalToken, LexicalTokenType, NewLexicalToken};
+use std::cell::RefCell;
+use std::fmt::Debug;
+use std::rc::Rc;
+use zero_parser::parser::{ParserContext, ParserResult};
+
+mod tokenizer;
+
+fn assert_lexical_parser(
+    except: LexicalToken,
+    parser: fn(
+        Rc<RefCell<ParserContext<char, ()>>>,
+        &[char],
+    ) -> ParserResult<char, NewLexicalToken>,
+    input: &str,
+) {
+    let context = ParserContext::new_with_str(input, ());
+    let borrowed_context = context.borrow();
+    let input = borrowed_context.input_slice();
+    let (_, token) = parser(context.clone(), input).unwrap();
+
+    assert_eq!(except, token);
+}
+
+fn assert_parser<T, F>(except: T, parser: F, input: &str)
+where
+    T: PartialEq + Debug,
+    F: Fn(Rc<RefCell<ParserContext<char, ()>>>, &[char]) -> ParserResult<char, T>,
+{
+    let context = ParserContext::new_with_str(input, ());
+    let borrowed_context = context.borrow();
+    let (_, token) = parser(context.clone(), borrowed_context.input_slice()).unwrap();
+
+    assert_eq!(except, token);
+}
+
+#[test]
+fn keyword_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Keyword,
+            literal_value: "const",
+        },
+        keyword_parser,
+        "const int a = 3;",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Keyword,
+            literal_value: "return",
+        },
+        keyword_parser,
+        "return 0;",
+    );
+}
+
+#[test]
+fn delimiter_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Delimiter,
+            literal_value: "{",
+        },
+        delimiter_parser,
+        "{ int i = 3;}",
+    );
+}
+
+#[test]
+fn operator_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Operator,
+            literal_value: "!=",
+        },
+        operator_parser,
+        "!=",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Operator,
+            literal_value: "!",
+        },
+        operator_parser,
+        "!",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Operator,
+            literal_value: ">=",
+        },
+        operator_parser,
+        ">=",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Operator,
+            literal_value: ">",
+        },
+        operator_parser,
+        "> 123",
+    );
+}
+
+#[test]
+fn identifier_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: "a",
+        },
+        identifier_parser,
+        "a = 3",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: "_123",
+        },
+        identifier_parser,
+        "_123 = NULL",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: "test_123",
+        },
+        identifier_parser,
+        "test_123 += 3",
+    );
+}
+
+#[test]
+fn decimal_integer_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(100),
+            literal_value: "100",
+        },
+        decimal_integer_parser,
+        "100",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(56),
+            literal_value: "56",
+        },
+        decimal_integer_parser,
+        "56 + 44",
+    );
+}
+
+#[test]
+fn octal_integer_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(63),
+            literal_value: "077",
+        },
+        octal_integer_parser,
+        "077",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(0),
+            literal_value: "0",
+        },
+        octal_integer_parser,
+        "0",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(0),
+            literal_value: "00",
+        },
+        octal_integer_parser,
+        "00",
+    );
+}
+
+#[test]
+fn hexadecimal_integer_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(0),
+            literal_value: "0x0",
+        },
+        hexadecimal_integer_parser,
+        "0x0",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(0),
+            literal_value: "0X00",
+        },
+        hexadecimal_integer_parser,
+        "0X00",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(15),
+            literal_value: "0xF",
+        },
+        hexadecimal_integer_parser,
+        "0xF",
+    );
+}
+
+#[test]
+fn integer_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(0),
+            literal_value: "0",
+        },
+        integer_parser,
+        "0",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(45),
+            literal_value: "0055",
+        },
+        integer_parser,
+        "0055",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(291),
+            literal_value: "0X123",
+        },
+        integer_parser,
+        "0X123",
+    );
+}
+
+#[test]
+fn float_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(100.0),
+            literal_value: "100.0",
+        },
+        float_parser,
+        "100.0",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(0.5),
+            literal_value: "0.5",
+        },
+        float_parser,
+        "0.5",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(123.456),
+            literal_value: "123.456",
+        },
+        float_parser,
+        "123.456",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(3.14),
+            literal_value: "03.14",
+        },
+        float_parser,
+        "03.14",
+    );
+}
+
+#[test]
+fn literal_string_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::LiteralString,
+            literal_value: "abc",
+        },
+        literal_string_parser,
+        "\"abc\"",
+    );
+}
+
+#[test]
+fn comments_test() {
+    assert_parser((), comments_parser, "//test while-if\n");
+    assert_parser((), comments_parser, "/* asasdasd */");
+}
+
+#[test]
+fn combine_parser_test() {
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstInteger(120),
+            literal_value: "120",
+        },
+        combine_parser,
+        "120",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::ConstFloat(120.11),
+            literal_value: "120.110",
+        },
+        combine_parser,
+        "120.110",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::LiteralString,
+            literal_value: "Hello, world!\n",
+        },
+        combine_parser,
+        "\"Hello, world!\n\"",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Keyword,
+            literal_value: "const",
+        },
+        combine_parser,
+        "const",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: "const_number",
+        },
+        combine_parser,
+        "const_number",
+    );
+
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Keyword,
+            literal_value: "int",
+        },
+        combine_parser,
+        "int",
+    );
+    assert_lexical_parser(
+        LexicalToken {
+            token_type: LexicalTokenType::Identifier,
+            literal_value: "int_a",
+        },
+        combine_parser,
+        "int_a",
+    );
+}