feat: 添加标准测试

2026-06-16 16:56:17 +08:00
parent a76cd79be2
commit 983a7d8b34
4 changed files with 1751 additions and 1 deletions
@@ -309,3 +309,327 @@ fn is_ident_start(c: char) -> bool {
 fn is_ident_part(c: char) -> bool {
    is_ident_start(c) || c.is_ascii_digit()
 }
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Helper: tokenize input and return only TokenKinds (drop EOF and positions).
+    fn tokenize(input: &str) -> Vec<TokenKind> {
+        let (tokens, _errors) = Lexer::new(input).tokenize();
+        tokens.into_iter().filter_map(|t| {
+            if matches!(t.kind, TokenKind::EOF) { None } else { Some(t.kind) }
+        }).collect()
+    }
+
+    /// Helper: tokenize and return full tokens (with positions).
+    fn tokenize_full(input: &str) -> (Vec<Token>, Vec<RuntimeError>) {
+        Lexer::new(input).tokenize()
+    }
+
+    // ----- single-character tokens -----
+
+    #[test]
+    fn single_char_tokens() {
+        let kinds = tokenize("(){},;.:");
+        assert_eq!(kinds, vec![
+            TokenKind::LeftParen, TokenKind::RightParen,
+            TokenKind::LeftBrace, TokenKind::RightBrace,
+            TokenKind::Comma, TokenKind::Semicolon,
+            TokenKind::Dot, TokenKind::Colon,
+        ]);
+    }
+
+    #[test]
+    fn brackets() {
+        let kinds = tokenize("[]");
+        assert_eq!(kinds, vec![TokenKind::LeftBracket, TokenKind::RightBracket]);
+    }
+
+    // ----- operators -----
+
+    #[test]
+    fn arithmetic_operators() {
+        let kinds = tokenize("+ - * / %");
+        assert_eq!(kinds, vec![
+            TokenKind::Plus, TokenKind::Minus, TokenKind::Star,
+            TokenKind::Slash, TokenKind::Percent,
+        ]);
+    }
+
+    #[test]
+    fn comparison_operators() {
+        let kinds = tokenize("< <= > >=");
+        assert_eq!(kinds, vec![
+            TokenKind::Less, TokenKind::LessEqual,
+            TokenKind::Greater, TokenKind::GreaterEqual,
+        ]);
+    }
+
+    #[test]
+    fn equality_operators() {
+        let kinds = tokenize("== !=");
+        assert_eq!(kinds, vec![TokenKind::EqualEqual, TokenKind::BangEqual]);
+    }
+
+    #[test]
+    fn logical_operators() {
+        let kinds = tokenize("&& || !");
+        assert_eq!(kinds, vec![TokenKind::AndAnd, TokenKind::OrOr, TokenKind::Bang]);
+    }
+
+    #[test]
+    fn assignment_operators() {
+        let kinds = tokenize("= += -= *= /= %=");
+        assert_eq!(kinds, vec![
+            TokenKind::Equal,
+            TokenKind::PlusEqual, TokenKind::MinusEqual,
+            TokenKind::StarEqual, TokenKind::SlashEqual,
+            TokenKind::PercentEqual,
+        ]);
+    }
+
+    #[test]
+    fn operator_proximity_no_spaces() {
+        let kinds = tokenize("a+b-c*d/e%f");
+        // identifiers a, b, c, d, e, f interleaved with operators
+        let ids: Vec<_> = kinds.iter().filter(|k| matches!(k, TokenKind::Identifier(_))).collect();
+        assert_eq!(ids.len(), 6);
+    }
+
+    // ----- numbers -----
+
+    #[test]
+    fn integer_literal() {
+        let kinds = tokenize("42");
+        assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
+    }
+
+    #[test]
+    fn float_literal() {
+        let kinds = tokenize("3.14");
+        assert_eq!(kinds, vec![TokenKind::Number(3.14)]);
+    }
+
+    #[test]
+    fn zero_and_negative() {
+        // Note: `-` is a separate unary operator, but `0` should lex correctly
+        let kinds = tokenize("0");
+        assert_eq!(kinds, vec![TokenKind::Number(0.0)]);
+    }
+
+    // ----- strings -----
+
+    #[test]
+    fn string_literal() {
+        let kinds = tokenize("\"hello\"");
+        assert_eq!(kinds, vec![TokenKind::String("hello".into())]);
+    }
+
+    #[test]
+    fn empty_string() {
+        let kinds = tokenize("\"\"");
+        assert_eq!(kinds, vec![TokenKind::String("".into())]);
+    }
+
+    #[test]
+    fn string_with_spaces() {
+        let kinds = tokenize("\"hello world\"");
+        assert_eq!(kinds, vec![TokenKind::String("hello world".into())]);
+    }
+
+    #[test]
+    fn multiple_strings() {
+        let kinds = tokenize("\"a\" \"b\"");
+        assert_eq!(kinds, vec![
+            TokenKind::String("a".into()),
+            TokenKind::String("b".into()),
+        ]);
+    }
+
+    // ----- identifiers and keywords -----
+
+    #[test]
+    fn identifiers() {
+        let kinds = tokenize("foo bar _private x1");
+        let ids: Vec<_> = kinds.iter().filter_map(|k| {
+            if let TokenKind::Identifier(s) = k { Some(s.clone()) } else { None }
+        }).collect();
+        assert_eq!(ids, vec!["foo", "bar", "_private", "x1"]);
+    }
+
+    #[test]
+    fn keywords() {
+        let kinds = tokenize("let fn if else while for break continue return true false nil");
+        assert_eq!(kinds, vec![
+            TokenKind::Let, TokenKind::Fn,
+            TokenKind::If, TokenKind::Else,
+            TokenKind::While, TokenKind::For,
+            TokenKind::Break, TokenKind::Continue,
+            TokenKind::Return,
+            TokenKind::True, TokenKind::False,
+            TokenKind::Nil,
+        ]);
+    }
+
+    #[test]
+    fn identifier_looks_like_keyword_prefix() {
+        // "lettuce" should be identifier, not "let" + error
+        let kinds = tokenize("lettuce");
+        assert_eq!(kinds, vec![TokenKind::Identifier("lettuce".into())]);
+    }
+
+    // ----- comments -----
+
+    #[test]
+    fn line_comment_ignored() {
+        let kinds = tokenize("// this is a comment\n42");
+        assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
+    }
+
+    #[test]
+    fn comment_at_end_of_file() {
+        let kinds = tokenize("42 // trailing comment");
+        assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
+    }
+
+    #[test]
+    fn comment_only_line() {
+        let kinds = tokenize("// just a comment");
+        assert!(kinds.is_empty());
+    }
+
+    #[test]
+    fn comment_between_tokens() {
+        let kinds = tokenize("1 // comment\n2");
+        assert_eq!(kinds, vec![TokenKind::Number(1.0), TokenKind::Number(2.0)]);
+    }
+
+    // ----- whitespace -----
+
+    #[test]
+    fn mixed_whitespace() {
+        let kinds = tokenize("  \t 42\r\n  84");
+        assert_eq!(kinds, vec![TokenKind::Number(42.0), TokenKind::Number(84.0)]);
+    }
+
+    #[test]
+    fn only_whitespace() {
+        let kinds = tokenize("   \n\t  \n  ");
+        assert!(kinds.is_empty());
+    }
+
+    // ----- errors -----
+
+    #[test]
+    fn unterminated_string() {
+        let (_, errors) = tokenize_full("\"no closing quote");
+        assert_eq!(errors.len(), 1);
+        assert!(errors[0].to_string().contains("Unterminated"));
+    }
+
+    #[test]
+    fn bare_ampersand() {
+        let (_, errors) = tokenize_full("&");
+        assert_eq!(errors.len(), 1);
+        assert!(errors[0].to_string().contains("Unexpected '&'"));
+    }
+
+    #[test]
+    fn bare_pipe() {
+        let (_, errors) = tokenize_full("|");
+        assert_eq!(errors.len(), 1);
+        assert!(errors[0].to_string().contains("Unexpected '|'"));
+    }
+
+    #[test]
+    fn unexpected_character() {
+        let (_, errors) = tokenize_full("@");
+        assert_eq!(errors.len(), 1);
+        assert!(errors[0].to_string().contains("Unexpected character"));
+    }
+
+    #[test]
+    fn error_does_not_block_rest() {
+        // Bare `&` produces an error, but tokenization continues
+        let (tokens, errors) = tokenize_full("& 42");
+        assert_eq!(errors.len(), 1);
+        let kinds: Vec<_> = tokens.iter().filter(|t| !matches!(t.kind, TokenKind::EOF)).map(|t| t.kind.clone()).collect();
+        assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
+    }
+
+    // ----- position tracking -----
+
+    #[test]
+    fn positions_single_line() {
+        let (tokens, _) = tokenize_full("let x = 5");
+        // let  @ 1:1
+        // x    @ 1:5
+        // =    @ 1:7
+        // 5    @ 1:9
+        // EOF
+        let token_kinds: Vec<_> = tokens.iter().map(|t| (&t.kind, t.line, t.column)).collect();
+        // Check positions for the first token (let)
+        assert_eq!(token_kinds[0], (&TokenKind::Let, 1, 1));
+        // Check positions for fourth token (5)
+        assert_eq!(token_kinds[3], (&TokenKind::Number(5.0), 1, 9));
+    }
+
+    #[test]
+    fn positions_multi_line() {
+        let (tokens, _) = tokenize_full("let\nx\n=");
+        // let @ 1:1
+        // x   @ 2:1
+        // =   @ 3:1
+        let token_kinds: Vec<_> = tokens.iter().map(|t| (t.line, t.column)).collect();
+        assert_eq!(token_kinds[0], (1, 1)); // let
+        assert_eq!(token_kinds[1], (2, 1)); // x
+        assert_eq!(token_kinds[2], (3, 1)); // =
+    }
+
+    #[test]
+    fn position_after_string() {
+        let (tokens, _) = tokenize_full("\"hi\" 42");
+        // "hi" @ 1:1
+        // 42   @ 1:6 (after "hi" which is 4 chars + space)
+        let token_kinds: Vec<_> = tokens.iter().map(|t| (t.line, t.column)).collect();
+        assert_eq!(token_kinds[0], (1, 1)); // "hi" (the opening quote position)
+        assert_eq!(token_kinds[1], (1, 6)); // 42
+    }
+
+    // ----- edge cases -----
+
+    #[test]
+    fn empty_input() {
+        let kinds = tokenize("");
+        assert!(kinds.is_empty());
+    }
+
+    #[test]
+    fn complex_expression() {
+        let kinds = tokenize("if (x >= 0 && y < 10) { return x + y; }");
+        assert_eq!(kinds, vec![
+            TokenKind::If,
+            TokenKind::LeftParen,
+            TokenKind::Identifier("x".into()),
+            TokenKind::GreaterEqual,
+            TokenKind::Number(0.0),
+            TokenKind::AndAnd,
+            TokenKind::Identifier("y".into()),
+            TokenKind::Less,
+            TokenKind::Number(10.0),
+            TokenKind::RightParen,
+            TokenKind::LeftBrace,
+            TokenKind::Return,
+            TokenKind::Identifier("x".into()),
+            TokenKind::Plus,
+            TokenKind::Identifier("y".into()),
+            TokenKind::Semicolon,
+            TokenKind::RightBrace,
+        ]);
+    }
+}