Struct TokenAndSpan 

pub struct TokenAndSpan {
    pub token: Token,
    pub had_line_break: bool,
    pub span: Span,
}

Fields

token: Token

had_line_break: bool

Had a line break before this token?

span: Span

Trait Implementations

impl Clone for TokenAndSpan

fn clone(&self) -> TokenAndSpan

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenAndSpan

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a> Lexer<'a, TokenAndSpan> for Lexer<'a>

type CommentsBuffer = CommentsBuffer

type State = State

type Token = Token

fn input(&self) -> &StringInput<'a>

fn input_mut(&mut self) -> &mut StringInput<'a>

fn push_error(&mut self, error: Error)

fn state(&self) -> &Self::State

fn state_mut(&mut self) -> &mut Self::State

fn comments(&self) -> Option<&'a dyn Comments>

fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>

fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>

unsafe fn input_slice(&mut self, start: BytePos, end: BytePos) -> &'a str

Safety

fn input_uncons_while(&mut self, f: impl FnMut(char) -> bool) -> &'a str

fn atom<'b>(&self, s: impl Into<Cow<'b, str>>) -> Atom

fn had_line_break_before_last(&self) -> bool

fn span(&self, start: BytePos) -> Span

fn bump(&mut self)

fn is(&self, c: u8) -> bool

fn is_str(&self, s: &str) -> bool

fn eat(&mut self, c: u8) -> bool

fn cur(&self) -> Option<char>

fn peek(&self) -> Option<char>

fn peek_ahead(&self) -> Option<char>

fn cur_pos(&self) -> BytePos

fn last_pos(&self) -> BytePos

fn error<T>(&self, start: BytePos, kind: SyntaxError) -> LexResult<T>

Shorthand for let span = self.span(start); self.error_span(span)

fn error_span<T>(&self, span: Span, kind: SyntaxError) -> LexResult<T>

fn emit_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_error_span(&mut self, span: Span, kind: SyntaxError)

fn emit_strict_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_module_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn skip_line_comment(&mut self, start_skip: usize)

fn skip_block_comment(&mut self)

Expects current char to be ‘/’ and next char to be ‘*’.

fn skip_space<const LEX_COMMENTS: bool>(&mut self)

Skip comments or whitespaces.

fn ensure_not_ident(&mut self) -> LexResult<()>

Ensure that ident cannot directly follow numbers.

fn make_legacy_octal(&mut self, start: BytePos, val: f64) -> LexResult<f64>

fn read_digits<F, Ret, const RADIX: u8>( &mut self, op: F, allow_num_separator: bool, has_underscore: &mut bool, ) -> LexResult<Ret>

where F: FnMut(Ret, u8, u32) -> LexResult<(Ret, bool)>, Ret: Copy + Default,

op- |total, radix, value| -> (total * radix + value, continue)

fn read_number_no_dot_as_str<const RADIX: u8>( &mut self, ) -> LexResult<LazyInteger>

This can read long integers like “13612536612375123612312312312312312312312”.

fn read_number<const START_WITH_DOT: bool, const START_WITH_ZERO: bool>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Reads an integer, octal integer, or floating-point number

fn read_int_u32<const RADIX: u8>(&mut self, len: u8) -> LexResult<Option<u32>>

fn read_radix_number<const RADIX: u8>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Returns Left(value) or Right(BigInt)

fn consume_pending_comments(&mut self)

Consume pending comments.

fn read_jsx_word(&mut self) -> LexResult<Self::Token>

Read a JSX identifier (valid tag or attribute name).

fn read_jsx_entity(&mut self) -> LexResult<(char, String)>

fn read_jsx_new_line( &mut self, normalize_crlf: bool, ) -> LexResult<Either<&'static str, char>>

fn read_jsx_str(&mut self, quote: char) -> LexResult<Self::Token>

fn read_unicode_escape(&mut self) -> LexResult<Vec<Char>>

fn read_shebang(&mut self) -> LexResult<Option<Atom>>

fn read_tmpl_token(&mut self, start_of_tpl: BytePos) -> LexResult<Self::Token>

fn read_escaped_char( &mut self, in_template: bool, ) -> LexResult<Option<Vec<Char>>>

Read an escaped character for string literal.

fn read_regexp(&mut self, start: BytePos) -> LexResult<Self::Token>

Expects current char to be ‘/’

fn read_word_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This method is optimized for texts without escape sequences.

fn read_word_as_str_with_slow_path( &mut self, slice_start: BytePos, ) -> LexResult<(Cow<'a, str>, bool)>

Slow path for identifier parsing that handles Unicode and escapes

fn read_token_number_sign(&mut self) -> LexResult<Self::Token>

#

fn read_token_dot(&mut self) -> LexResult<Self::Token>

Read a token given ..

fn read_token_question_mark(&mut self) -> LexResult<Self::Token>

Read a token given ?.

fn read_token_colon(&mut self) -> LexResult<Self::Token>

Read a token given :.

fn read_token_zero(&mut self) -> LexResult<Self::Token>

Read a token given 0.

fn read_token_logical<const C: u8>(&mut self) -> LexResult<Self::Token>

Read a token given | or &.

fn read_token_mul_mod(&mut self, is_mul: bool) -> LexResult<Self::Token>

Read a token given * or %.

fn read_slash(&mut self) -> LexResult<Self::Token>

fn read_ident_unknown(&mut self) -> LexResult<Self::Token>

This can be used if there’s no keyword starting with the first character.

fn read_str_lit(&mut self) -> LexResult<Self::Token>

See https://tc39.github.io/ecma262/#sec-literals-string-literals

fn read_keyword_with( &mut self, convert: &dyn Fn(&str) -> Option<Self::Token>, ) -> LexResult<Self::Token>

fn read_keyword_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This is a performant version of Lexer::read_word_as_str_with for reading keywords. We should make sure the first byte is a valid ASCII.

impl NextTokenAndSpan for TokenAndSpan

type Token = Token

fn token(&self) -> &Self::Token

fn span(&self) -> Span

fn had_line_break(&self) -> bool

impl PartialEq for TokenAndSpan

fn eq(&self, other: &TokenAndSpan) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Spanned for TokenAndSpan

fn span(&self) -> Span

Get span of self.

fn span_lo(&self) -> BytePos

fn span_hi(&self) -> BytePos

impl TokenAndSpan for TokenAndSpan

type Token = Token

fn new(token: Token, span: Span, had_line_break: bool) -> Self

fn token(&self) -> &Token

fn take_token(self) -> Token

fn had_line_break(&self) -> bool

fn span(&self) -> Span

impl<'a, I: Tokens<TokenAndSpan>> TokenFactory<'a, TokenAndSpan, I> for Token

const ABSTRACT: Self

const ACCESSOR: Self

const ANY: Self

const ARROW: Self = Self::Arrow

const AS: Self

const ASSERT: Self

const ASSERTS: Self

const ASYNC: Self

const AT: Self = Self::At

const AWAIT: Self

const BACKQUOTE: Self = Self::BackQuote

const BANG: Self = Self::Bang

const BIGINT: Self

const BIT_AND: Self

const BIT_AND_EQ: Self

const BIT_OR: Self

const BIT_OR_EQ: Self

const BOOLEAN: Self

const BREAK: Self

const CASE: Self

const CATCH: Self

const CLASS: Self

const COLON: Self = Self::Colon

const COMMA: Self = Self::Comma

const CONST: Self

const CONTINUE: Self

const DEBUGGER: Self

const DECLARE: Self

const DEFAULT: Self

const DELETE: Self

const DIV: Self

const DIV_EQ: Self

const DO: Self

const DOLLAR_LBRACE: Self = Self::DollarLBrace

const DOT: Self = Self::Dot

const DOTDOTDOT: Self = Self::DotDotDot

const ELSE: Self

const ENUM: Self

const EOF: Self = Token::Eof

const EQUAL: Self

const EXP: Self

const EXPORT: Self

const EXP_EQ: Self

const EXTENDS: Self

const FALSE: Self

const FINALLY: Self

const FOR: Self

const FROM: Self

const FUNCTION: Self

const GET: Self

const GLOBAL: Self

const GREATER: Self

const GREATER_EQ: Self

const HASH: Self = Self::Hash

const IF: Self

const IMPLEMENTS: Self

const IMPORT: Self

const IN: Self

const INFER: Self

const INSTANCEOF: Self

const INTERFACE: Self

const INTRINSIC: Self

const IS: Self

const JSX_TAG_END: Self = Self::JSXTagEnd

const JSX_TAG_START: Self = Self::JSXTagStart

const KEYOF: Self

const LBRACE: Self = Self::LBrace

const LBRACKET: Self = Self::LBracket

const LESS: Self

const LESS_EQ: Self

const LET: Self

const LOGICAL_AND: Self

const LOGICAL_AND_EQ: Self

const LOGICAL_OR: Self

const LOGICAL_OR_EQ: Self

const LPAREN: Self = Self::LParen

const LSHIFT: Self

const LSHIFT_EQ: Self

const MINUS: Self

const MINUS_MINUS: Self = Self::MinusMinus

const MOD: Self

const MOD_EQ: Self

const MUL: Self

const MUL_EQ: Self

const NAMESPACE: Self

const NEVER: Self

const NEW: Self

const NULL: Self

const NULLISH_ASSIGN: Self

const NULLISH_COALESCING: Self

const NUMBER: Self

const OBJECT: Self

const OF: Self

const PACKAGE: Self

const PLUS: Self

const PLUS_PLUS: Self = Self::PlusPlus

const PRIVATE: Self

const PROTECTED: Self

const PUBLIC: Self

const QUESTION: Self = Token::QuestionMark

const RBRACE: Self = Self::RBrace

const RBRACKET: Self = Self::RBracket

const READONLY: Self

const REQUIRE: Self

const RETURN: Self

const RPAREN: Self = Self::RParen

const RSHIFT: Self

const RSHIFT_EQ: Self

const SATISFIES: Self

const SEMI: Self = Self::Semi

const SET: Self

const STATIC: Self

const STRING: Self

const SUPER: Self

const SWITCH: Self

const SYMBOL: Self

const TARGET: Self

const THIS: Self

const THROW: Self

const TILDE: Self = Self::Tilde

const TRUE: Self

const TRY: Self

const TYPE: Self

const TYPEOF: Self

const UNDEFINED: Self

const UNIQUE: Self

const UNKNOWN: Self

const USING: Self

const VAR: Self

const VOID: Self

const WHILE: Self

const WITH: Self

const YIELD: Self

const ZERO_FILL_RSHIFT: Self

const ZERO_FILL_RSHIFT_EQ: Self

type Buffer = Buffer<I>

type Lexer = Lexer<'a>

fn jsx_name(name: &'a str, lexer: &mut Lexer<'a>) -> Self

fn is_jsx_name(&self) -> bool

fn take_jsx_name(self, _: &mut Self::Buffer) -> Atom

fn str(value: Atom, raw: Atom, _: &mut Lexer<'a>) -> Self

fn template(cooked: LexResult<Atom>, raw: Atom, _: &mut Lexer<'a>) -> Self

fn regexp(content: Atom, flags: Atom, _: &mut Lexer<'a>) -> Self

fn num(value: f64, raw: Atom, _: &mut Lexer<'a>) -> Self

fn bigint(value: Box<BigIntValue>, raw: Atom, _: &mut Lexer<'a>) -> Self

fn unknown_ident(value: Atom, _: &mut Lexer<'a>) -> Self

fn is_word(&self) -> bool

fn is_reserved(&self, ctx: Context) -> bool

fn into_atom(self, _: &mut Lexer<'a>) -> Option<Atom>

fn is_error(&self) -> bool

fn take_error(self, _: &mut Self::Buffer) -> Error

fn is_str(&self) -> bool

fn is_str_raw_content(&self, content: &str, _: &Self::Buffer) -> bool

fn take_str(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn is_num(&self) -> bool

fn take_num(self, _: &mut Self::Buffer) -> (f64, Atom)

fn is_bigint(&self) -> bool

fn take_bigint(self, _: &mut Self::Buffer) -> (Box<BigIntValue>, Atom)

fn take_word(self, _: &Self::Buffer) -> Option<Atom>

fn is_unknown_ident(&self) -> bool

fn take_unknown_ident(self, _: &mut Self::Buffer) -> Atom

fn take_unknown_ident_ref<'b>(&'b self, _: &'b Self::Buffer) -> &'b Atom

fn is_keyword(&self) -> bool

fn is_known_ident(&self) -> bool

fn take_known_ident(&self) -> Atom

fn is_regexp(&self) -> bool

fn is_template(&self) -> bool

fn take_template(self, _: &mut Self::Buffer) -> (LexResult<Atom>, Atom)

fn jsx_text(value: Atom, raw: Atom, _: &mut Self::Lexer) -> Self

fn is_jsx_text(&self) -> bool

fn take_jsx_text(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn starts_expr(&self) -> bool

fn to_string(&self, _: &Self::Buffer) -> String

fn is_bin_op(&self) -> bool

fn as_assign_op(&self) -> Option<AssignOp>

fn as_bin_op(&self) -> Option<BinaryOp>

fn follows_keyword_let(&self) -> bool

fn is_assign_op(&self) -> bool

fn take_regexp(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn shebang(value: Atom, _: &mut Self::Lexer) -> Self

fn is_shebang(&self) -> bool

fn take_shebang(self, _: &mut Self::Buffer) -> Atom

fn is_no_substitution_template_literal(&self) -> bool

fn is_template_head(&self) -> bool

fn is_less(&self) -> bool

fn is_less_eq(&self) -> bool

fn is_greater(&self) -> bool

fn is_colon(&self) -> bool

fn is_comma(&self) -> bool

fn is_equal(&self) -> bool

fn is_question(&self) -> bool

fn is_null(&self) -> bool

fn is_lshift(&self) -> bool

fn is_rshift(&self) -> bool

fn is_rshift_eq(&self) -> bool

fn is_greater_eq(&self) -> bool

fn is_true(&self) -> bool

fn is_false(&self) -> bool

fn is_enum(&self) -> bool

fn is_yield(&self) -> bool

fn is_let(&self) -> bool

fn is_var(&self) -> bool

fn is_static(&self) -> bool

fn is_extends(&self) -> bool

fn is_implements(&self) -> bool

fn is_interface(&self) -> bool

fn is_type(&self) -> bool

fn is_package(&self) -> bool

fn is_private(&self) -> bool

fn is_protected(&self) -> bool

fn is_public(&self) -> bool

fn is_readonly(&self) -> bool

fn is_await(&self) -> bool

fn is_break(&self) -> bool

fn is_continue(&self) -> bool

fn is_arrow(&self) -> bool

fn is_this(&self) -> bool

fn is_super(&self) -> bool

fn is_using(&self) -> bool

fn is_backquote(&self) -> bool

fn is_lparen(&self) -> bool

fn is_rparen(&self) -> bool

fn is_lbracket(&self) -> bool

fn is_rbracket(&self) -> bool

fn is_lbrace(&self) -> bool

fn is_rbrace(&self) -> bool

fn is_function(&self) -> bool

fn is_class(&self) -> bool

fn is_if(&self) -> bool

fn is_return(&self) -> bool

fn is_switch(&self) -> bool

fn is_throw(&self) -> bool

fn is_catch(&self) -> bool

fn is_finally(&self) -> bool

fn is_try(&self) -> bool

fn is_with(&self) -> bool

fn is_while(&self) -> bool

fn is_new(&self) -> bool

fn is_ident_ref(&self, ctx: Context) -> bool

fn is_import(&self) -> bool

fn is_export(&self) -> bool

fn is_dot(&self) -> bool

fn is_do(&self) -> bool

fn is_for(&self) -> bool

fn is_from(&self) -> bool

fn is_dotdotdot(&self) -> bool

fn is_plus(&self) -> bool

fn is_minus(&self) -> bool

fn is_bang(&self) -> bool

fn is_tilde(&self) -> bool

fn is_plus_plus(&self) -> bool

fn is_minus_minus(&self) -> bool

fn is_delete(&self) -> bool

fn is_typeof(&self) -> bool

fn is_of(&self) -> bool

fn is_void(&self) -> bool

fn is_hash(&self) -> bool

fn is_in(&self) -> bool

fn is_const(&self) -> bool

fn is_star(&self) -> bool

fn is_mod(&self) -> bool

fn is_semi(&self) -> bool

fn is_slash(&self) -> bool

fn is_slash_eq(&self) -> bool

fn is_jsx_tag_start(&self) -> bool

fn is_jsx_tag_end(&self) -> bool

fn is_asserts(&self) -> bool

fn is_is(&self) -> bool

fn is_as(&self) -> bool

fn is_satisfies(&self) -> bool

fn is_instanceof(&self) -> bool

fn is_async(&self) -> bool

fn is_case(&self) -> bool

fn is_default(&self) -> bool

fn is_debugger(&self) -> bool

fn is_bit_and(&self) -> bool

fn is_bit_or(&self) -> bool

fn is_exp(&self) -> bool

fn is_eof(&self) -> bool

impl<I: Tokens<TokenAndSpan>> Tokens<TokenAndSpan> for Capturing<I>

type Checkpoint = <I as Tokens<TokenAndSpan>>::Checkpoint

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx_mut(&mut self) -> &mut Context

fn ctx(&self) -> Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, allow: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl Tokens<TokenAndSpan> for Lexer<'_>

type Checkpoint = Lexer<'_>

fn set_ctx(&mut self, ctx: Context)

fn ctx(&self) -> Context

fn ctx_mut(&mut self) -> &mut Context

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, allow: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl Tokens<TokenAndSpan> for TokensInput

type Checkpoint = TokensInput

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx_mut(&mut self) -> &mut Context

fn ctx(&self) -> Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, _: bool)

fn set_next_regexp(&mut self, _: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl StructuralPartialEq for TokenAndSpan

Layout

Note: Most layout information is completely unstable and may even differ between compilations. The only exception is types with certain repr(...) attributes. Please see the Rust Reference's “Type Layout” chapter for details on type layout guarantees.

Size: 40 bytes