Function lex

pub fn lex(source: &str) -> Lexer<'_> 

Lex a Lustre file into localized Tokens

Shorthand for Lexer::from_source.

Example

// This iterator contains the tokens and their respective range
let spanned_tokens = lex("type register = bool^32;");

// We only keep the tokens and filter out trivia (spaces, comments)
let tokens = spanned_tokens.map(|(token, _range)| token).filter(|t| t.is_non_trivia());

assert_eq!(tokens.collect::<Vec<_>>(), [
    Token::Type,
    Token::Ident,
    Token::Equal,
    Token::Bool,
    Token::Hat,
    Token::IConst,
    Token::Semicolon,
]);