Type Definition chumsky::stream::BoxStream

pub type BoxStream<'a, I, S> = Stream<'a, I, S, Box<dyn Iterator<Item = (I, S)> + 'a>>;

A Stream that pulls tokens from a boxed Iterator.

Implementations

impl<'a, I: Clone, S: Span + 'a> BoxStream<'a, I, S>

pub fn from_nested<P: 'a, Iter: Iterator<Item = (P, S)>, Many: Iterator<Item = (P, S)>, F: FnMut((P, S)) -> Flat<(I, S), Many> + 'a>( eoi: S, iter: Iter, flatten: F ) -> Self

Create a new Stream from an iterator of nested tokens and a function that flattens them.

It’s not uncommon for compilers to perform delimiter parsing during the lexing stage (Rust does this!). When this is done, the output of the lexing stage is usually a series of nested token trees. This functions allows you to easily flatten such token trees into a linear token stream so that they can be parsed (Chumsky currently only support parsing linear streams of inputs).

For reference, here is syn’s TokenTree type that it uses when parsing Rust syntax.

Examples

type Span = std::ops::Range<usize>;

fn span_at(at: usize) -> Span { at..at + 1 }

#[derive(Clone)]
enum Token {
    Local(String),
    Int(i64),
    Bool(bool),
    Add,
    Sub,
    OpenParen,
    CloseParen,
    OpenBrace,
    CloseBrace,
    // etc.
}

enum Delimiter {
    Paren, // ( ... )
    Brace, // { ... }
}

// The structure of this token tree is very similar to that which Rust uses.
// See: https://docs.rs/syn/0.11.1/syn/enum.TokenTree.html
enum TokenTree {
    Token(Token),
    Tree(Delimiter, Vec<(TokenTree, Span)>),
}

// A function that turns a series of nested token trees into a linear stream that can be used for parsing.
fn flatten_tts(eoi: Span, token_trees: Vec<(TokenTree, Span)>) -> BoxStream<'static, Token, Span> {
    use std::iter::once;
    // Currently, this is quite an explicit process: it will likely become easier in future versions of Chumsky.
    Stream::from_nested(
        eoi,
        token_trees.into_iter(),
        |(tt, span)| match tt {
            // For token trees that contain just a single token, no flattening needs to occur!
            TokenTree::Token(token) => Flat::Single((token, span)),
            // Flatten a parenthesised token tree into an iterator of the inner token trees, surrounded by parenthesis tokens
            TokenTree::Tree(Delimiter::Paren, tree) => Flat::Many(once((TokenTree::Token(Token::OpenParen), span_at(span.start)))
                .chain(tree.into_iter())
                .chain(once((TokenTree::Token(Token::CloseParen), span_at(span.end - 1))))),
            // Flatten a braced token tree into an iterator of the inner token trees, surrounded by brace tokens
            TokenTree::Tree(Delimiter::Brace, tree) => Flat::Many(once((TokenTree::Token(Token::OpenBrace), span_at(span.start)))
                .chain(tree.into_iter())
                .chain(once((TokenTree::Token(Token::CloseBrace), span_at(span.end - 1))))),
        }
    )
}