elmma/aster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 抽离测试代码

Browse Source
This commit is contained in:
0264408
2026-06-16 19:25:01 +08:00
parent fdd42fce19
commit e4272943aa
6 changed files with 2494 additions and 2507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lexer/lexer.rs
+2 -413
View File
@@ -346,417 +346,6 @@ fn is_ident_part(c: char) -> bool {
is_ident_start(c) || c.is_ascii_digit()
}
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
/// Helper: tokenize input and return only TokenKinds (drop EOF and positions).
fn tokenize(input: &str) -> Vec<TokenKind> {
let (tokens, _errors) = Lexer::new(input).tokenize();
tokens.into_iter().filter_map(|t| {
if matches!(t.kind, TokenKind::EOF) { None } else { Some(t.kind) }
}).collect()
}
/// Helper: tokenize and return full tokens (with positions).
fn tokenize_full(input: &str) -> (Vec<Token>, Vec<RuntimeError>) {
Lexer::new(input).tokenize()
}
// ----- single-character tokens -----
#[test]
fn single_char_tokens() {
let kinds = tokenize("(){},;.:");
assert_eq!(kinds, vec![
TokenKind::LeftParen, TokenKind::RightParen,
TokenKind::LeftBrace, TokenKind::RightBrace,
TokenKind::Comma, TokenKind::Semicolon,
TokenKind::Dot, TokenKind::Colon,
]);
}
#[test]
fn brackets() {
let kinds = tokenize("[]");
assert_eq!(kinds, vec![TokenKind::LeftBracket, TokenKind::RightBracket]);
}
// ----- operators -----
#[test]
fn arithmetic_operators() {
let kinds = tokenize("+ - * / %");
assert_eq!(kinds, vec![
TokenKind::Plus, TokenKind::Minus, TokenKind::Star,
TokenKind::Slash, TokenKind::Percent,
]);
}
#[test]
fn comparison_operators() {
let kinds = tokenize("< <= > >=");
assert_eq!(kinds, vec![
TokenKind::Less, TokenKind::LessEqual,
TokenKind::Greater, TokenKind::GreaterEqual,
]);
}
#[test]
fn equality_operators() {
let kinds = tokenize("== !=");
assert_eq!(kinds, vec![TokenKind::EqualEqual, TokenKind::BangEqual]);
}
#[test]
fn logical_operators() {
let kinds = tokenize("&& || !");
assert_eq!(kinds, vec![TokenKind::AndAnd, TokenKind::OrOr, TokenKind::Bang]);
}
#[test]
fn assignment_operators() {
let kinds = tokenize("= += -= *= /= %=");
assert_eq!(kinds, vec![
TokenKind::Equal,
TokenKind::PlusEqual, TokenKind::MinusEqual,
TokenKind::StarEqual, TokenKind::SlashEqual,
TokenKind::PercentEqual,
]);
}
#[test]
fn operator_proximity_no_spaces() {
let kinds = tokenize("a+b-c*d/e%f");
// identifiers a, b, c, d, e, f interleaved with operators
let ids: Vec<_> = kinds.iter().filter(|k| matches!(k, TokenKind::Identifier(_))).collect();
assert_eq!(ids.len(), 6);
}
// ----- numbers -----
#[test]
fn integer_literal() {
let kinds = tokenize("42");
assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
}
#[test]
fn float_literal() {
let kinds = tokenize("3.14");
assert_eq!(kinds, vec![TokenKind::Number(3.14)]);
}
#[test]
fn zero_and_negative() {
// Note: `-` is a separate unary operator, but `0` should lex correctly
let kinds = tokenize("0");
assert_eq!(kinds, vec![TokenKind::Number(0.0)]);
}
// ----- strings -----
#[test]
fn string_literal() {
let kinds = tokenize("\"hello\"");
assert_eq!(kinds, vec![TokenKind::String("hello".into())]);
}
#[test]
fn empty_string() {
let kinds = tokenize("\"\"");
assert_eq!(kinds, vec![TokenKind::String("".into())]);
}
#[test]
fn string_with_spaces() {
let kinds = tokenize("\"hello world\"");
assert_eq!(kinds, vec![TokenKind::String("hello world".into())]);
}
#[test]
fn multiple_strings() {
let kinds = tokenize("\"a\" \"b\"");
assert_eq!(kinds, vec![
TokenKind::String("a".into()),
TokenKind::String("b".into()),
]);
}
#[test]
fn string_escape_newline() {
let kinds = tokenize("\"a\\nb\"");
assert_eq!(kinds, vec![TokenKind::String("a\nb".into())]);
}
#[test]
fn string_escape_tab() {
let kinds = tokenize("\"a\\tb\"");
assert_eq!(kinds, vec![TokenKind::String("a\tb".into())]);
}
#[test]
fn string_escape_quote() {
let kinds = tokenize("\"he\\\"llo\"");
assert_eq!(kinds, vec![TokenKind::String("he\"llo".into())]);
}
#[test]
fn string_escape_backslash() {
let kinds = tokenize("\"a\\\\b\"");
assert_eq!(kinds, vec![TokenKind::String("a\\b".into())]);
}
#[test]
fn string_escape_carriage_return() {
let kinds = tokenize("\"a\\rb\"");
assert_eq!(kinds, vec![TokenKind::String("a\rb".into())]);
}
#[test]
fn string_unknown_escape_is_error() {
let (_, errors) = tokenize_full("\"\\x\"");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unknown escape"));
}
// ----- single-quoted strings -----
#[test]
fn single_quoted_string() {
let kinds = tokenize("'hello'");
assert_eq!(kinds, vec![TokenKind::String("hello".into())]);
}
#[test]
fn single_quoted_empty_string() {
let kinds = tokenize("''");
assert_eq!(kinds, vec![TokenKind::String("".into())]);
}
#[test]
fn single_quoted_with_escape() {
let kinds = tokenize("'a\\'b'");
assert_eq!(kinds, vec![TokenKind::String("a'b".into())]);
}
#[test]
fn single_quoted_with_newline_escape() {
let kinds = tokenize("'a\\nb'");
assert_eq!(kinds, vec![TokenKind::String("a\nb".into())]);
}
#[test]
fn mixed_quote_strings() {
let kinds = tokenize("\"double\" 'single'");
assert_eq!(kinds, vec![
TokenKind::String("double".into()),
TokenKind::String("single".into()),
]);
}
#[test]
fn single_quote_can_contain_double_quote() {
let kinds = tokenize("'he\"llo'");
assert_eq!(kinds, vec![TokenKind::String("he\"llo".into())]);
}
#[test]
fn double_quote_can_contain_single_quote() {
let kinds = tokenize("\"it's\"");
assert_eq!(kinds, vec![TokenKind::String("it's".into())]);
}
#[test]
fn unterminated_single_quoted_string() {
let (_, errors) = tokenize_full("'unterminated");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unterminated"));
}
// ----- identifiers and keywords -----
#[test]
fn identifiers() {
let kinds = tokenize("foo bar _private x1");
let ids: Vec<_> = kinds.iter().filter_map(|k| {
if let TokenKind::Identifier(s) = k { Some(s.clone()) } else { None }
}).collect();
assert_eq!(ids, vec!["foo", "bar", "_private", "x1"]);
}
#[test]
fn keywords() {
let kinds = tokenize("let const fn if else while for in break continue return true false nil");
assert_eq!(kinds, vec![
TokenKind::Let, TokenKind::Const, TokenKind::Fn,
TokenKind::If, TokenKind::Else,
TokenKind::While, TokenKind::For, TokenKind::In,
TokenKind::Break, TokenKind::Continue,
TokenKind::Return,
TokenKind::True, TokenKind::False,
TokenKind::Nil,
]);
}
#[test]
fn identifier_looks_like_keyword_prefix() {
// "lettuce" should be identifier, not "let" + error
let kinds = tokenize("lettuce");
assert_eq!(kinds, vec![TokenKind::Identifier("lettuce".into())]);
}
// ----- comments -----
#[test]
fn line_comment_ignored() {
let kinds = tokenize("// this is a comment\n42");
assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
}
#[test]
fn comment_at_end_of_file() {
let kinds = tokenize("42 // trailing comment");
assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
}
#[test]
fn comment_only_line() {
let kinds = tokenize("// just a comment");
assert!(kinds.is_empty());
}
#[test]
fn comment_between_tokens() {
let kinds = tokenize("1 // comment\n2");
assert_eq!(kinds, vec![TokenKind::Number(1.0), TokenKind::Number(2.0)]);
}
// ----- whitespace -----
#[test]
fn mixed_whitespace() {
let kinds = tokenize(" \t 42\r\n 84");
assert_eq!(kinds, vec![TokenKind::Number(42.0), TokenKind::Number(84.0)]);
}
#[test]
fn only_whitespace() {
let kinds = tokenize(" \n\t \n ");
assert!(kinds.is_empty());
}
// ----- errors -----
#[test]
fn unterminated_string() {
let (_, errors) = tokenize_full("\"no closing quote");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unterminated"));
}
#[test]
fn bare_ampersand() {
let (_, errors) = tokenize_full("&");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unexpected '&'"));
}
#[test]
fn bare_pipe() {
let (_, errors) = tokenize_full("|");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unexpected '|'"));
}
#[test]
fn unexpected_character() {
let (_, errors) = tokenize_full("@");
assert_eq!(errors.len(), 1);
assert!(errors[0].to_string().contains("Unexpected character"));
}
#[test]
fn error_does_not_block_rest() {
// Bare `&` produces an error, but tokenization continues
let (tokens, errors) = tokenize_full("& 42");
assert_eq!(errors.len(), 1);
let kinds: Vec<_> = tokens.iter().filter(|t| !matches!(t.kind, TokenKind::EOF)).map(|t| t.kind.clone()).collect();
assert_eq!(kinds, vec![TokenKind::Number(42.0)]);
}
// ----- position tracking -----
#[test]
fn positions_single_line() {
let (tokens, _) = tokenize_full("let x = 5");
// let @ 1:1
// x @ 1:5
// = @ 1:7
// 5 @ 1:9
// EOF
let token_kinds: Vec<_> = tokens.iter().map(|t| (&t.kind, t.line, t.column)).collect();
// Check positions for the first token (let)
assert_eq!(token_kinds[0], (&TokenKind::Let, 1, 1));
// Check positions for fourth token (5)
assert_eq!(token_kinds[3], (&TokenKind::Number(5.0), 1, 9));
}
#[test]
fn positions_multi_line() {
let (tokens, _) = tokenize_full("let\nx\n=");
// let @ 1:1
// x @ 2:1
// = @ 3:1
let token_kinds: Vec<_> = tokens.iter().map(|t| (t.line, t.column)).collect();
assert_eq!(token_kinds[0], (1, 1)); // let
assert_eq!(token_kinds[1], (2, 1)); // x
assert_eq!(token_kinds[2], (3, 1)); // =
}
#[test]
fn position_after_string() {
let (tokens, _) = tokenize_full("\"hi\" 42");
// "hi" @ 1:1
// 42 @ 1:6 (after "hi" which is 4 chars + space)
let token_kinds: Vec<_> = tokens.iter().map(|t| (t.line, t.column)).collect();
assert_eq!(token_kinds[0], (1, 1)); // "hi" (the opening quote position)
assert_eq!(token_kinds[1], (1, 6)); // 42
}
// ----- edge cases -----
#[test]
fn empty_input() {
let kinds = tokenize("");
assert!(kinds.is_empty());
}
#[test]
fn complex_expression() {
let kinds = tokenize("if (x >= 0 && y < 10) { return x + y; }");
assert_eq!(kinds, vec![
TokenKind::If,
TokenKind::LeftParen,
TokenKind::Identifier("x".into()),
TokenKind::GreaterEqual,
TokenKind::Number(0.0),
TokenKind::AndAnd,
TokenKind::Identifier("y".into()),
TokenKind::Less,
TokenKind::Number(10.0),
TokenKind::RightParen,
TokenKind::LeftBrace,
TokenKind::Return,
TokenKind::Identifier("x".into()),
TokenKind::Plus,
TokenKind::Identifier("y".into()),
TokenKind::Semicolon,
TokenKind::RightBrace,
]);
}
}
#[path = "tests.rs"]
mod tests;