lrlex/
lexer.rs

1use std::{
2    collections::{HashMap, HashSet},
3    fmt::Debug,
4    hash::Hash,
5    marker::PhantomData,
6    slice::Iter,
7    str::FromStr,
8};
9
10use cfgrammar::{
11    NewlineCache, Span,
12    header::{GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Value},
13    span::Location,
14};
15use num_traits::{AsPrimitive, PrimInt, Unsigned};
16use regex::{Regex, RegexBuilder};
17
18use lrpar::{Lexeme, Lexer, LexerTypes, NonStreamingLexer};
19
20use crate::{
21    LRLexError, LexBuildError, LexBuildResult, StartStateId,
22    parser::{LexParser, StartState, StartStateOperation},
23};
24
25#[doc(hidden)]
26/// Corresponds to the options for `regex::RegexBuilder`.
27#[derive(Clone, Debug)]
28#[non_exhaustive]
29pub struct LexFlags {
30    // The following values when `None` grmtools provides default values for in `DEFAULT_LEX_FLAGS`
31    pub dot_matches_new_line: Option<bool>,
32    pub multi_line: Option<bool>,
33    pub octal: Option<bool>,
34    pub posix_escapes: Option<bool>,
35    pub allow_wholeline_comments: Option<bool>,
36
37    // All the following values when `None` default to the `regex` crate's default value.
38    pub case_insensitive: Option<bool>,
39    pub swap_greed: Option<bool>,
40    pub ignore_whitespace: Option<bool>,
41    pub unicode: Option<bool>,
42    pub size_limit: Option<usize>,
43    pub dfa_size_limit: Option<usize>,
44    pub nest_limit: Option<u32>,
45}
46
47impl<T: Clone> TryFrom<&mut Header<T>> for LexFlags {
48    type Error = HeaderError<T>;
49    fn try_from(header: &mut Header<T>) -> Result<LexFlags, HeaderError<T>> {
50        use cfgrammar::header::Setting;
51        let mut lex_flags = UNSPECIFIED_LEX_FLAGS;
52        let LexFlags {
53            dot_matches_new_line,
54            multi_line,
55            octal,
56            posix_escapes,
57            allow_wholeline_comments,
58            case_insensitive,
59            swap_greed,
60            ignore_whitespace,
61            unicode,
62            size_limit,
63            dfa_size_limit,
64            nest_limit,
65        } = &mut lex_flags;
66        macro_rules! cvt_flag {
67            ($it:ident) => {
68                header.mark_used(&stringify!($it).to_string());
69                *$it = match header.get(stringify!($it)) {
70                    Some(HeaderValue(_, Value::Flag(flag, _))) => Some(*flag),
71                    Some(HeaderValue(loc, _)) => Err(HeaderError {
72                        kind: HeaderErrorKind::ConversionError("LexFlags", "Expected boolean"),
73                        locations: vec![loc.clone()],
74                    })?,
75                    None => None,
76                }
77            };
78        }
79        cvt_flag!(dot_matches_new_line);
80        cvt_flag!(multi_line);
81        cvt_flag!(octal);
82        cvt_flag!(posix_escapes);
83        cvt_flag!(allow_wholeline_comments);
84        cvt_flag!(case_insensitive);
85        cvt_flag!(swap_greed);
86        cvt_flag!(ignore_whitespace);
87        cvt_flag!(unicode);
88        macro_rules! cvt_num {
89            ($it:ident, $num_ty: ty) => {
90                header.mark_used(&stringify!($it).to_string());
91                *$it = match header.get(stringify!($it)) {
92                    Some(HeaderValue(_, Value::Setting(Setting::Num(n, _)))) => Some(*n as $num_ty),
93                    Some(HeaderValue(loc, _)) => Err(HeaderError {
94                        kind: HeaderErrorKind::ConversionError("LexFlags", "Expected numeric"),
95                        locations: vec![loc.clone()],
96                    })?,
97                    None => None,
98                }
99            };
100        }
101        cvt_num!(size_limit, usize);
102        cvt_num!(dfa_size_limit, usize);
103        cvt_num!(nest_limit, u32);
104        Ok(lex_flags)
105    }
106}
107
108impl From<&LexFlags> for Header<Location> {
109    fn from(flags: &LexFlags) -> Header<Location> {
110        let mut header = Header::new();
111        let LexFlags {
112            dot_matches_new_line,
113            multi_line,
114            octal,
115            posix_escapes,
116            allow_wholeline_comments,
117            case_insensitive,
118            swap_greed,
119            ignore_whitespace,
120            unicode,
121            size_limit,
122            dfa_size_limit,
123            nest_limit,
124        } = flags;
125        macro_rules! cvt_flag {
126            ($it: ident) => {
127                $it.map(|x| {
128                    header.insert(
129                        stringify!($it).to_string(),
130                        HeaderValue(
131                            Location::Other("From<&LexFlags".to_string()),
132                            Value::Flag(x, Location::Other("From<&LexFlags>".to_string())),
133                        ),
134                    )
135                });
136            };
137        }
138        cvt_flag!(dot_matches_new_line);
139        cvt_flag!(multi_line);
140        cvt_flag!(octal);
141        cvt_flag!(posix_escapes);
142        cvt_flag!(allow_wholeline_comments);
143        cvt_flag!(case_insensitive);
144        cvt_flag!(swap_greed);
145        cvt_flag!(ignore_whitespace);
146        cvt_flag!(unicode);
147
148        macro_rules! cvt_num {
149            ($it: ident) => {
150                $it.map(|x| {
151                    use cfgrammar::header::Setting;
152                    header.insert(
153                        stringify!($it).to_string(),
154                        HeaderValue(
155                            Location::Other("From<&LexFlags".to_string()),
156                            Value::Setting(Setting::Num(
157                                x as u64,
158                                Location::Other("From<&LexFlags>".to_string()),
159                            )),
160                        ),
161                    )
162                });
163            };
164        }
165        cvt_num!(size_limit);
166        cvt_num!(dfa_size_limit);
167        cvt_num!(nest_limit);
168
169        header
170    }
171}
172
173/// LexFlags with flags set to default values.
174#[doc(hidden)]
175pub const DEFAULT_LEX_FLAGS: LexFlags = LexFlags {
176    allow_wholeline_comments: Some(false),
177    dot_matches_new_line: Some(true),
178    multi_line: Some(true),
179    octal: Some(true),
180    posix_escapes: Some(false),
181    case_insensitive: None,
182    ignore_whitespace: None,
183    swap_greed: None,
184    unicode: None,
185    size_limit: None,
186    dfa_size_limit: None,
187    nest_limit: None,
188};
189
190#[doc(hidden)]
191/// LexFlags with all of the values `None`.
192pub const UNSPECIFIED_LEX_FLAGS: LexFlags = LexFlags {
193    allow_wholeline_comments: None,
194    dot_matches_new_line: None,
195    multi_line: None,
196    octal: None,
197    posix_escapes: None,
198    case_insensitive: None,
199    ignore_whitespace: None,
200    swap_greed: None,
201    unicode: None,
202    size_limit: None,
203    dfa_size_limit: None,
204    nest_limit: None,
205};
206
207#[derive(Debug, Clone)]
208#[doc(hidden)]
209pub struct Rule<StorageT> {
210    /// If `Some`, this specifies the ID that lexemes resulting from this rule will have. Note that
211    /// lrlex gives rules a guaranteed unique value by default, though users can later override
212    /// that, potentially undermining uniqueness if they're not careful.
213    ///
214    /// If `None`, then this rule specifies lexemes which should not appear in the user's input.
215    pub(super) tok_id: Option<StorageT>,
216    /// This rule's name. If None, then text which matches this rule will be skipped (i.e. will not
217    /// create a lexeme).
218    #[deprecated(note = "Use the name() function")]
219    pub name: Option<String>,
220    #[deprecated(note = "Use the name_span() function")]
221    pub name_span: Span,
222    pub(super) re_str: String,
223    re: Regex,
224    /// Id(s) of permitted start conditions for the lexer to match this rule.
225    #[deprecated(note = "Use the start_states() function")]
226    pub start_states: Vec<usize>,
227    /// If Some(_), successful matching of this rule will cause the current stack of start
228    /// conditions in the lexer to be updated with the enclosed value, using the designated
229    /// operation.
230    /// If None, successful matching causes no change to the current start condition.
231    #[deprecated(note = "Use the target_state() function")]
232    pub target_state: Option<(usize, StartStateOperation)>,
233}
234
235impl<StorageT: PrimInt> Rule<StorageT> {
236    /// Create a new `Rule`. This interface is unstable and should only be used by code generated
237    /// by lrlex itself.
238    #[doc(hidden)]
239    #[allow(private_interfaces)]
240    #[allow(clippy::too_many_arguments)]
241    pub fn new(
242        _: crate::unstable_api::InternalPublicApi,
243        tok_id: Option<StorageT>,
244        name: Option<String>,
245        name_span: Span,
246        re_str: String,
247        start_states: Vec<usize>,
248        target_state: Option<(usize, StartStateOperation)>,
249        lex_flags: &LexFlags,
250    ) -> Result<Rule<StorageT>, regex::Error> {
251        let mut re = RegexBuilder::new(&format!("\\A(?:{})", re_str));
252        let mut re = re
253            .octal(lex_flags.octal.unwrap())
254            .multi_line(lex_flags.multi_line.unwrap())
255            .dot_matches_new_line(lex_flags.dot_matches_new_line.unwrap());
256
257        if let Some(flag) = lex_flags.ignore_whitespace {
258            re = re.ignore_whitespace(flag)
259        }
260        if let Some(flag) = lex_flags.unicode {
261            re = re.unicode(flag)
262        }
263        if let Some(flag) = lex_flags.case_insensitive {
264            re = re.case_insensitive(flag)
265        }
266        if let Some(flag) = lex_flags.swap_greed {
267            re = re.swap_greed(flag)
268        }
269        if let Some(sz) = lex_flags.size_limit {
270            re = re.size_limit(sz)
271        }
272        if let Some(sz) = lex_flags.dfa_size_limit {
273            re = re.dfa_size_limit(sz)
274        }
275        if let Some(lim) = lex_flags.nest_limit {
276            re = re.nest_limit(lim)
277        }
278
279        let re = re.build()?;
280        #[allow(deprecated)]
281        Ok(Rule {
282            tok_id,
283            name,
284            name_span,
285            re_str,
286            re,
287            start_states,
288            target_state,
289        })
290    }
291
292    /// Return this rule's token ID, if any.
293    ///
294    /// If `Some`, this specifies the ID that lexemes resulting from this rule will have. If
295    /// `None`, then this rule specifies lexemes which should not appear in the user's input.
296    pub fn tok_id(&self) -> Option<StorageT> {
297        self.tok_id
298    }
299
300    /// Return this rule's name. If `None`, then text which matches this rule will be skipped (i.e.
301    /// it will not result in the creation of a [Lexeme]).
302    pub fn name(&self) -> Option<&str> {
303        #[allow(deprecated)]
304        self.name.as_deref()
305    }
306
307    /// Return the [Span] of this rule's name.
308    pub fn name_span(&self) -> Span {
309        #[allow(deprecated)]
310        self.name_span
311    }
312
313    /// Return the original regular expression specified by the user for this [Rule].
314    pub fn re_str(&self) -> &str {
315        &self.re_str
316    }
317
318    /// Return the IDs of the permitted start conditions for the lexer to match this rule.
319    pub fn start_states(&self) -> &[usize] {
320        #[allow(deprecated)]
321        self.start_states.as_slice()
322    }
323
324    /// Return the IDs of the permitted start conditions for the lexer to match this rule.
325    pub fn target_state(&self) -> Option<(usize, StartStateOperation)> {
326        #[allow(deprecated)]
327        self.target_state.clone()
328    }
329}
330
331/// Methods which all lexer definitions must implement.
332pub trait LexerDef<LexerTypesT: LexerTypes>
333where
334    usize: AsPrimitive<LexerTypesT::StorageT>,
335{
336    #[doc(hidden)]
337    /// Instantiate a lexer from a set of `Rule`s. This is only intended to be used by compiled
338    /// lexers (see `ctbuilder.rs`).
339    fn from_rules(start_states: Vec<StartState>, rules: Vec<Rule<LexerTypesT::StorageT>>) -> Self
340    where
341        Self: Sized;
342
343    /// Instantiate a lexer from a string (e.g. representing a `.l` file).
344    fn from_str(s: &str) -> LexBuildResult<Self>
345    where
346        Self: Sized;
347
348    /// Get the `Rule` at index `idx`.
349    fn get_rule(&self, idx: usize) -> Option<&Rule<LexerTypesT::StorageT>>;
350
351    /// Get the `Rule` instance associated with a particular lexeme ID. Panics if no such rule
352    /// exists.
353    fn get_rule_by_id(&self, tok_id: LexerTypesT::StorageT) -> &Rule<LexerTypesT::StorageT>;
354
355    /// Get the `Rule` instance associated with a particular name.
356    fn get_rule_by_name(&self, n: &str) -> Option<&Rule<LexerTypesT::StorageT>>;
357
358    /// Set the id attribute on rules to the corresponding value in `map`. This is typically used
359    /// to synchronise a parser's notion of lexeme IDs with the lexers. While doing this, it keeps
360    /// track of which lexemes:
361    ///   1) are defined in the lexer but not referenced by the parser
362    ///   2) and referenced by the parser but not defined in the lexer
363    ///
364    /// and returns them as a tuple `(Option<HashSet<&str>>, Option<HashSet<&str>>)` in the order
365    /// (*defined_in_lexer_missing_from_parser*, *referenced_in_parser_missing_from_lexer*). Since
366    /// in most cases both sets are expected to be empty, `None` is returned to avoid a `HashSet`
367    /// allocation.
368    ///
369    /// Lexing and parsing can continue if either set is non-empty, so it is up to the caller as to
370    /// what action they take if either return set is non-empty. A non-empty set #1 is often
371    /// benign: some lexers deliberately define tokens which are not used (e.g. reserving future
372    /// keywords). A non-empty set #2 is more likely to be an error since there are parts of the
373    /// grammar where nothing the user can input will be parseable.
374    fn set_rule_ids<'a>(
375        &'a mut self,
376        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
377    ) -> (Option<HashSet<&'a str>>, Option<HashSet<&'a str>>);
378
379    fn set_rule_ids_spanned<'a>(
380        &'a mut self,
381        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
382    ) -> (Option<HashSet<&'a str>>, Option<HashSet<(&'a str, Span)>>);
383
384    /// Returns an iterator over all rules in this AST.
385    fn iter_rules(&self) -> Iter<'_, Rule<LexerTypesT::StorageT>>;
386
387    /// Returns an iterator over all start states in this AST.
388    fn iter_start_states(&self) -> Iter<'_, StartState>;
389}
390
391/// This struct represents, in essence, a .l file in memory. From it one can produce an
392/// [LRNonStreamingLexer] which actually lexes inputs.
393#[derive(Debug, Clone)]
394pub struct LRNonStreamingLexerDef<LexerTypesT: LexerTypes>
395where
396    usize: AsPrimitive<LexerTypesT::StorageT>,
397{
398    rules: Vec<Rule<LexerTypesT::StorageT>>,
399    start_states: Vec<StartState>,
400    lex_flags: LexFlags,
401    pub(crate) expected_missing_tokens: Vec<String>,
402    phantom: PhantomData<LexerTypesT>,
403}
404
405impl<LexerTypesT: LexerTypes> LexerDef<LexerTypesT> for LRNonStreamingLexerDef<LexerTypesT>
406where
407    usize: AsPrimitive<LexerTypesT::StorageT>,
408    LexerTypesT::StorageT: TryFrom<usize>,
409{
410    fn from_rules(
411        start_states: Vec<StartState>,
412        rules: Vec<Rule<LexerTypesT::StorageT>>,
413    ) -> LRNonStreamingLexerDef<LexerTypesT> {
414        LRNonStreamingLexerDef {
415            rules,
416            start_states,
417            lex_flags: DEFAULT_LEX_FLAGS,
418            expected_missing_tokens: vec![],
419            phantom: PhantomData,
420        }
421    }
422
423    /// Given a `.l` file in an `&str`, returns a `LrNonStreamingLexerDef`
424    /// after merging the `%grmtools` section with the default set of `LexFlags`.
425    fn from_str(s: &str) -> LexBuildResult<LRNonStreamingLexerDef<LexerTypesT>> {
426        let (mut header, pos) = GrmtoolsSectionParser::new(s, false)
427            .parse()
428            .map_err(|mut errs| errs.drain(..).map(LexBuildError::from).collect::<Vec<_>>())?;
429        let flags = LexFlags::try_from(&mut header).map_err(|e| vec![e.into()])?;
430        LexParser::<LexerTypesT>::new_with_lex_flags(s[pos..].to_string(), flags.clone()).map(|p| {
431            LRNonStreamingLexerDef {
432                rules: p.rules,
433                start_states: p.start_states,
434                lex_flags: flags,
435                expected_missing_tokens: p.expected_missing_tokens,
436                phantom: PhantomData,
437            }
438        })
439    }
440
441    fn get_rule(&self, idx: usize) -> Option<&Rule<LexerTypesT::StorageT>> {
442        self.rules.get(idx)
443    }
444
445    fn get_rule_by_id(&self, tok_id: LexerTypesT::StorageT) -> &Rule<LexerTypesT::StorageT> {
446        self.rules
447            .iter()
448            .find(|r| r.tok_id == Some(tok_id))
449            .unwrap()
450    }
451
452    fn get_rule_by_name(&self, n: &str) -> Option<&Rule<LexerTypesT::StorageT>> {
453        self.rules.iter().find(|r| r.name() == Some(n))
454    }
455
456    fn set_rule_ids<'a>(
457        &'a mut self,
458        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
459    ) -> (Option<HashSet<&'a str>>, Option<HashSet<&'a str>>) {
460        let (missing_from_parser, missing_from_lexer) = self.set_rule_ids_spanned(rule_ids_map);
461        let missing_from_lexer =
462            missing_from_lexer.map(|missing| missing.iter().map(|(name, _)| *name).collect());
463        (missing_from_parser, missing_from_lexer)
464    }
465
466    fn set_rule_ids_spanned<'a>(
467        &'a mut self,
468        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
469    ) -> (Option<HashSet<&'a str>>, Option<HashSet<(&'a str, Span)>>) {
470        // Because we have to iter_mut over self.rules, we can't easily store a reference to the
471        // rule's name at the same time. Instead, we store the index of each such rule and
472        // recover the names later. This has the unfortunate consequence of extended the mutable
473        // borrow for the rest of the 'a lifetime. To avoid that we could return idx's here.
474        // But the original `set_rule_ids` invalidates indexes.  In the spirit of keeping that
475        // behavior consistent, this also returns the span.
476        let mut missing_from_parser_idxs = Vec::new();
477        let mut rules_with_names = 0;
478        for (i, r) in self.rules.iter_mut().enumerate() {
479            if let Some(n) = r.name() {
480                match rule_ids_map.get(n) {
481                    Some(tok_id) => r.tok_id = Some(*tok_id),
482                    None => {
483                        r.tok_id = None;
484                        missing_from_parser_idxs.push(i);
485                    }
486                }
487                rules_with_names += 1;
488            }
489        }
490
491        let missing_from_parser = if missing_from_parser_idxs.is_empty() {
492            None
493        } else {
494            let mut mfp = HashSet::with_capacity(missing_from_parser_idxs.len());
495            for i in &missing_from_parser_idxs {
496                mfp.insert((self.rules[*i].name().unwrap(), self.rules[*i].name_span()));
497            }
498            Some(mfp)
499        };
500
501        let missing_from_lexer =
502            if rules_with_names - missing_from_parser_idxs.len() == rule_ids_map.len() {
503                None
504            } else {
505                Some(
506                    rule_ids_map
507                        .keys()
508                        .cloned()
509                        .collect::<HashSet<&str>>()
510                        .difference(
511                            &self
512                                .rules
513                                .iter()
514                                .filter_map(|x| x.name())
515                                .collect::<HashSet<&str>>(),
516                        )
517                        .cloned()
518                        .collect::<HashSet<&str>>(),
519                )
520            };
521
522        (missing_from_lexer, missing_from_parser)
523    }
524
525    fn iter_rules(&self) -> Iter<'_, Rule<LexerTypesT::StorageT>> {
526        self.rules.iter()
527    }
528
529    fn iter_start_states(&self) -> Iter<'_, StartState> {
530        self.start_states.iter()
531    }
532}
533
534impl<
535    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
536    LexerTypesT: LexerTypes<StorageT = StorageT>,
537> LRNonStreamingLexerDef<LexerTypesT>
538where
539    usize: AsPrimitive<StorageT>,
540    LexerTypesT::StorageT: TryFrom<usize>,
541{
542    /// Uses the `lex_flags` passed in ignoring any settings in the `%grmtools` section.
543    pub fn new_with_options(
544        s: &str,
545        lex_flags: LexFlags,
546    ) -> LexBuildResult<LRNonStreamingLexerDef<LexerTypesT>> {
547        let (_, pos) = GrmtoolsSectionParser::new(s, false).parse().unwrap();
548        LexParser::<LexerTypesT>::new_with_lex_flags(s[pos..].to_string(), lex_flags.clone()).map(
549            |p| LRNonStreamingLexerDef {
550                rules: p.rules,
551                start_states: p.start_states,
552                lex_flags,
553                expected_missing_tokens: p.expected_missing_tokens,
554                phantom: PhantomData,
555            },
556        )
557    }
558
559    /// Return an [LRNonStreamingLexer] for the `String` `s` that will lex relative to this
560    /// [LRNonStreamingLexerDef].
561    pub fn lexer<'lexer, 'input: 'lexer>(
562        &'lexer self,
563        s: &'input str,
564    ) -> LRNonStreamingLexer<'lexer, 'input, LexerTypesT> {
565        let mut lexemes = vec![];
566        let mut i = 0;
567        let mut state_stack: Vec<(usize, &StartState)> = Vec::new();
568        let initial_state = match self.get_start_state_by_id(0) {
569            None => {
570                lexemes.push(Err(LRLexError::new(Span::new(i, i))));
571                return LRNonStreamingLexer::new(s, lexemes, NewlineCache::from_str(s).unwrap());
572            }
573            Some(state) => state,
574        };
575        state_stack.push((1, initial_state));
576
577        while i < s.len() {
578            let old_i = i;
579            let mut longest = 0; // Length of the longest match
580            let mut longest_ridx = 0; // This is only valid iff longest != 0
581            let current_state = match state_stack.last() {
582                None => {
583                    lexemes.push(Err(LRLexError::new(Span::new(i, i))));
584                    return LRNonStreamingLexer::new(
585                        s,
586                        lexemes,
587                        NewlineCache::from_str(s).unwrap(),
588                    );
589                }
590                Some((_, s)) => s,
591            };
592            for (ridx, r) in self.iter_rules().enumerate() {
593                if !Self::state_matches(current_state, r.start_states()) {
594                    continue;
595                }
596                if let Some(m) = r.re.find(&s[old_i..]) {
597                    let len = m.end();
598                    // Note that by using ">", we implicitly prefer an earlier over a later rule, if
599                    // both match an input of the same length.
600                    if len > longest {
601                        longest = len;
602                        longest_ridx = ridx;
603                    }
604                }
605            }
606            if longest > 0 {
607                let r = self.get_rule(longest_ridx).unwrap();
608                if r.name().is_some() {
609                    match r.tok_id {
610                        Some(tok_id) => {
611                            lexemes.push(Ok(Lexeme::new(tok_id, old_i, longest)));
612                        }
613                        None => {
614                            lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
615                            break;
616                        }
617                    }
618                }
619                if let Some((target_state_id, op)) = &r.target_state() {
620                    let state = match self.get_start_state_by_id(*target_state_id) {
621                        None => {
622                            // TODO: I can see an argument for lexing state to be either `None` or `Some(target_state_id)` here
623                            lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
624                            break;
625                        }
626                        Some(state) => state,
627                    };
628                    let head = state_stack.last_mut();
629                    match op {
630                        StartStateOperation::ReplaceStack => {
631                            state_stack.clear();
632                            state_stack.push((1, state));
633                        }
634                        StartStateOperation::Push => match head {
635                            Some((count, s)) if s.id == state.id => *count += 1,
636                            _ => state_stack.push((1, state)),
637                        },
638                        StartStateOperation::Pop => match head {
639                            Some((count, _s)) if *count > 1 => {
640                                *count -= 1;
641                            }
642                            Some(_) => {
643                                state_stack.pop();
644                                if state_stack.is_empty() {
645                                    state_stack.push((1, initial_state));
646                                }
647                            }
648                            None => {
649                                lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
650                                break;
651                            }
652                        },
653                    }
654                }
655                i += longest;
656            } else {
657                lexemes.push(Err(LRLexError::new_with_lexing_state(
658                    Span::new(old_i, old_i),
659                    StartStateId::new(current_state.id),
660                )));
661                break;
662            }
663        }
664        LRNonStreamingLexer::new(s, lexemes, NewlineCache::from_str(s).unwrap())
665    }
666
667    fn state_matches(state: &StartState, rule_states: &[usize]) -> bool {
668        if rule_states.is_empty() {
669            !state.exclusive
670        } else {
671            rule_states.contains(&state.id)
672        }
673    }
674
675    fn get_start_state_by_id(&self, id: usize) -> Option<&StartState> {
676        self.start_states.iter().find(|state| state.id == id)
677    }
678
679    /// Returns the final `LexFlags` used for this lex source
680    /// after all forced and default flags have been resolved.
681    pub(crate) fn lex_flags(&self) -> Option<&LexFlags> {
682        Some(&self.lex_flags)
683    }
684}
685
686/// An `LRNonStreamingLexer` holds a reference to a string and can lex it into [lrpar::Lexeme]s.
687/// Although the struct is tied to a single string, no guarantees are made about whether the
688/// lexemes are cached or not.
689pub struct LRNonStreamingLexer<'lexer, 'input: 'lexer, LexerTypesT: LexerTypes>
690where
691    usize: AsPrimitive<LexerTypesT::StorageT>,
692    LexerTypesT::StorageT: 'static + Debug + PrimInt,
693{
694    s: &'input str,
695    lexemes: Vec<Result<LexerTypesT::LexemeT, LRLexError>>,
696    newlines: NewlineCache,
697    phantom: PhantomData<(&'lexer (), LexerTypesT::StorageT)>,
698}
699
700impl<
701    'lexer,
702    'input: 'lexer,
703    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
704    LexerTypesT: LexerTypes<StorageT = StorageT>,
705> LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
706where
707    usize: AsPrimitive<StorageT>,
708{
709    /// Create a new `LRNonStreamingLexer` that read in: the input `s`; and derived `lexemes` and
710    /// `newlines`.
711    ///
712    /// Note that if one or more lexemes or newlines was not created from `s`, subsequent calls to
713    /// the `LRNonStreamingLexer` may cause `panic`s.
714    pub fn new(
715        s: &'input str,
716        lexemes: Vec<Result<LexerTypesT::LexemeT, LRLexError>>,
717        newlines: NewlineCache,
718    ) -> LRNonStreamingLexer<'lexer, 'input, LexerTypesT> {
719        LRNonStreamingLexer {
720            s,
721            lexemes,
722            newlines,
723            phantom: PhantomData,
724        }
725    }
726}
727
728impl<
729    'lexer,
730    'input: 'lexer,
731    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
732    LexerTypesT: LexerTypes<StorageT = StorageT, LexErrorT = LRLexError>,
733> Lexer<LexerTypesT> for LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
734where
735    usize: AsPrimitive<StorageT>,
736{
737    fn iter<'a>(
738        &'a self,
739    ) -> Box<dyn Iterator<Item = Result<LexerTypesT::LexemeT, LexerTypesT::LexErrorT>> + 'a> {
740        Box::new(self.lexemes.iter().cloned())
741    }
742}
743
744impl<'lexer, 'input: 'lexer, LexerTypesT: LexerTypes<LexErrorT = LRLexError>>
745    NonStreamingLexer<'input, LexerTypesT> for LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
746where
747    usize: AsPrimitive<LexerTypesT::StorageT>,
748{
749    fn span_str(&self, span: Span) -> &'input str {
750        if span.end() > self.s.len() {
751            panic!(
752                "Span {:?} exceeds known input length {}",
753                span,
754                self.s.len()
755            );
756        }
757        &self.s[span.start()..span.end()]
758    }
759
760    fn span_lines_str(&self, span: Span) -> &'input str {
761        debug_assert!(span.end() >= span.start());
762        if span.end() > self.s.len() {
763            panic!(
764                "Span {:?} exceeds known input length {}",
765                span,
766                self.s.len()
767            );
768        }
769
770        let (st, en) = self.newlines.span_line_bytes(span);
771        &self.s[st..en]
772    }
773
774    fn line_col(&self, span: Span) -> ((usize, usize), (usize, usize)) {
775        debug_assert!(span.end() >= span.start());
776        if span.end() > self.s.len() {
777            panic!(
778                "Span {:?} exceeds known input length {}",
779                span,
780                self.s.len()
781            );
782        }
783
784        (
785            self.newlines
786                .byte_to_line_num_and_col_num(self.s, span.start())
787                .unwrap(),
788            self.newlines
789                .byte_to_line_num_and_col_num(self.s, span.end())
790                .unwrap(),
791        )
792    }
793}
794
795#[cfg(test)]
796mod test {
797    use super::*;
798    use crate::{DefaultLexeme, DefaultLexerTypes};
799    use lrpar::LexError;
800    use std::collections::HashMap;
801
802    #[test]
803    fn test_basic() {
804        let src = r"
805%%
806[0-9]+ 'int'
807[a-zA-Z]+ 'id'
808[ \t] ;"
809            .to_string();
810        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
811        let mut map = HashMap::new();
812        map.insert("int", 0);
813        map.insert("id", 1);
814        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
815
816        let lexemes = lexerdef
817            .lexer("abc 123")
818            .iter()
819            .map(|x| x.unwrap())
820            .collect::<Vec<_>>();
821        assert_eq!(lexemes.len(), 2);
822        let lex1 = lexemes[0];
823        assert_eq!(lex1.tok_id(), 1u8);
824        assert_eq!(lex1.span().start(), 0);
825        assert_eq!(lex1.span().len(), 3);
826        let lex2 = lexemes[1];
827        assert_eq!(lex2.tok_id(), 0);
828        assert_eq!(lex2.span().start(), 4);
829        assert_eq!(lex2.span().len(), 3);
830    }
831
832    #[test]
833    fn test_posix_escapes() {
834        let src = r#"%%
835\\ 'slash'
836\a 'alert'
837\b 'backspace'
838\f 'feed'
839\n 'newline'
840\r 'return'
841\t 'tab'
842\v 'vtab'
843\q 'normal_char'
844"#
845        .to_string();
846        let mut options = DEFAULT_LEX_FLAGS;
847        options.posix_escapes = Some(true);
848        let lexerdef =
849            LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::new_with_options(&src, options)
850                .unwrap();
851        let lexemes = lexerdef
852            .lexer("\\\x07\x08\x0c\n\r\t\x0bq")
853            .iter()
854            .map(|x| x.unwrap())
855            .collect::<Vec<_>>();
856        assert_eq!(lexemes.len(), 9);
857        for i in 0..9u8 {
858            let lexeme = lexemes[i as usize];
859            assert_eq!(lexeme.tok_id(), i);
860        }
861    }
862
863    #[test]
864    fn test_non_posix_escapes() {
865        let src = r#"%%
866\\ 'slash'
867\a 'alert'
868a\b a 'work_break'
869\f 'feed'
870\n 'newline'
871\r 'return'
872\t 'tab'
873\v 'vtab'
874\q 'normal_char'
875"#
876        .to_string();
877        let mut options = DEFAULT_LEX_FLAGS;
878        options.posix_escapes = Some(false);
879        let lexerdef =
880            LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::new_with_options(&src, options)
881                .unwrap();
882        let lexemes = lexerdef
883            .lexer("\\\x07a a\x0c\n\r\t\x0bq")
884            .iter()
885            .map(|x| x.unwrap())
886            .collect::<Vec<_>>();
887        assert_eq!(lexemes.len(), 9);
888        for i in 0..9u8 {
889            let lexeme = lexemes[i as usize];
890            assert_eq!(lexeme.tok_id(), i);
891        }
892    }
893
894    #[test]
895    fn test_basic_error() {
896        let src = "
897%%
898[0-9]+ 'int'"
899            .to_string();
900        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
901        match lexerdef.lexer("abc").iter().next().unwrap() {
902            Ok(_) => panic!("Invalid input lexed"),
903            Err(e) => {
904                if e.span().start() != 0 || e.span().end() != 0 {
905                    panic!("Incorrect span returned {:?}", e.span());
906                }
907            }
908        };
909    }
910
911    #[test]
912    fn test_longest_match() {
913        let src = "%%
914if 'IF'
915[a-z]+ 'ID'
916[ ] ;"
917            .to_string();
918        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
919        let mut map = HashMap::new();
920        map.insert("IF", 0);
921        map.insert("ID", 1);
922        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
923
924        let lexemes = lexerdef
925            .lexer("iff if")
926            .iter()
927            .map(|x| x.unwrap())
928            .collect::<Vec<DefaultLexeme<u8>>>();
929        assert_eq!(lexemes.len(), 2);
930        let lex1 = lexemes[0];
931        assert_eq!(lex1.tok_id(), 1u8);
932        assert_eq!(lex1.span().start(), 0);
933        assert_eq!(lex1.span().len(), 3);
934        let lex2 = lexemes[1];
935        assert_eq!(lex2.tok_id(), 0);
936        assert_eq!(lex2.span().start(), 4);
937        assert_eq!(lex2.span().len(), 2);
938    }
939
940    #[test]
941    fn test_multibyte() {
942        let src = "%%
943[a❤]+ 'ID'
944[ ] ;"
945            .to_string();
946        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
947        let mut map = HashMap::new();
948        map.insert("ID", 0u8);
949        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
950
951        let lexer = lexerdef.lexer("a ❤ a");
952        let lexemes = lexer
953            .iter()
954            .map(|x| x.unwrap())
955            .collect::<Vec<DefaultLexeme<u8>>>();
956        assert_eq!(lexemes.len(), 3);
957        let lex1 = lexemes[0];
958        assert_eq!(lex1.span().start(), 0);
959        assert_eq!(lex1.span().len(), 1);
960        assert_eq!(lexer.span_str(lex1.span()), "a");
961        let lex2 = lexemes[1];
962        assert_eq!(lex2.span().start(), 2);
963        assert_eq!(lex2.span().len(), 3);
964        assert_eq!(lexer.span_str(lex2.span()), "❤");
965        let lex3 = lexemes[2];
966        assert_eq!(lex3.span().start(), 6);
967        assert_eq!(lex3.span().len(), 1);
968        assert_eq!(lexer.span_str(lex3.span()), "a");
969    }
970
971    #[test]
972    fn test_line_col() {
973        let src = "%%
974[a-z]+ 'ID'
975[ \\n] ;"
976            .to_string();
977        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
978        let mut map = HashMap::new();
979        map.insert("ID", 0u8);
980        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
981
982        let lexer = lexerdef.lexer("a b c");
983        let lexemes = lexer
984            .iter()
985            .map(|x| x.unwrap())
986            .collect::<Vec<DefaultLexeme<u8>>>();
987        assert_eq!(lexemes.len(), 3);
988        assert_eq!(lexer.line_col(lexemes[1].span()), ((1, 3), (1, 4)));
989        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a b c");
990        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "a b c");
991
992        let lexer = lexerdef.lexer("a b c\n");
993        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
994        assert_eq!(lexemes.len(), 3);
995        assert_eq!(lexer.line_col(lexemes[1].span()), ((1, 3), (1, 4)));
996        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a b c");
997        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "a b c");
998
999        let lexer = lexerdef.lexer(" a\nb\n  c d");
1000        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
1001        assert_eq!(lexemes.len(), 4);
1002        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 2), (1, 3)));
1003        assert_eq!(lexer.line_col(lexemes[1].span()), ((2, 1), (2, 2)));
1004        assert_eq!(lexer.line_col(lexemes[2].span()), ((3, 3), (3, 4)));
1005        assert_eq!(lexer.line_col(lexemes[3].span()), ((3, 5), (3, 6)));
1006        assert_eq!(lexer.span_lines_str(lexemes[0].span()), " a");
1007        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "b");
1008        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "  c d");
1009        assert_eq!(lexer.span_lines_str(lexemes[3].span()), "  c d");
1010
1011        let mut s = Vec::new();
1012        let mut offs = vec![0];
1013        for i in 0..71 {
1014            offs.push(offs[i] + i + 1);
1015            s.push(vec!["a"; i].join(" "));
1016        }
1017        let s = s.join("\n");
1018        let lexer = lexerdef.lexer(&s);
1019        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
1020        assert_eq!(lexemes.len(), offs[70]);
1021        assert_eq!(lexer.span_lines_str(Span::new(0, 0)), "");
1022        assert_eq!(lexer.span_lines_str(Span::new(0, 2)), "\na");
1023        assert_eq!(lexer.span_lines_str(Span::new(0, 4)), "\na\na a");
1024        assert_eq!(lexer.span_lines_str(Span::new(0, 7)), "\na\na a\na a a");
1025        assert_eq!(lexer.span_lines_str(Span::new(4, 7)), "a a\na a a");
1026        assert_eq!(lexer.span_lines_str(lexemes[0].span()), "a");
1027        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a a");
1028        assert_eq!(lexer.span_lines_str(lexemes[3].span()), "a a a");
1029        for i in 0..70 {
1030            assert_eq!(
1031                lexer.span_lines_str(lexemes[offs[i]].span()),
1032                vec!["a"; i + 1].join(" ")
1033            );
1034        }
1035    }
1036
1037    #[test]
1038    fn test_line_col_multibyte() {
1039        let src = "%%
1040[a-z❤]+ 'ID'
1041[ \\n] ;"
1042            .to_string();
1043        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1044        let mut map = HashMap::new();
1045        map.insert("ID", 0u8);
1046        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1047
1048        let lexer = lexerdef.lexer(" a\n❤ b");
1049        let lexemes = lexer
1050            .iter()
1051            .map(|x| x.unwrap())
1052            .collect::<Vec<DefaultLexeme<u8>>>();
1053        assert_eq!(lexemes.len(), 3);
1054        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 2), (1, 3)));
1055        assert_eq!(lexer.line_col(lexemes[1].span()), ((2, 1), (2, 2)));
1056        assert_eq!(lexer.line_col(lexemes[2].span()), ((2, 3), (2, 4)));
1057        assert_eq!(lexer.span_lines_str(lexemes[0].span()), " a");
1058        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "❤ b");
1059        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "❤ b");
1060    }
1061
1062    #[test]
1063    #[should_panic]
1064    fn test_bad_line_col() {
1065        let src = "%%
1066[a-z]+ 'ID'
1067[ \\n] ;"
1068            .to_string();
1069        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1070        let mut map = HashMap::new();
1071        map.insert("ID", 0u8);
1072        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1073
1074        let lexer = lexerdef.lexer("a b c");
1075
1076        lexer.line_col(Span::new(100, 100));
1077    }
1078
1079    #[test]
1080    fn test_missing_from_lexer_and_parser() {
1081        let src = "%%
1082[a-z]+ 'ID'
1083[ \\n] ;"
1084            .to_string();
1085        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1086        let mut map = HashMap::new();
1087        map.insert("INT", 0u8);
1088        let mut missing_from_lexer = HashSet::new();
1089        missing_from_lexer.insert("INT");
1090        let mut missing_from_parser = HashSet::new();
1091        missing_from_parser.insert("ID");
1092        assert_eq!(
1093            lexerdef.set_rule_ids(&map),
1094            (Some(missing_from_lexer), Some(missing_from_parser))
1095        );
1096
1097        match lexerdef.lexer(" a ").iter().next().unwrap() {
1098            Ok(_) => panic!("Invalid input lexed"),
1099            Err(e) => {
1100                if e.span().start() != 1 || e.span().end() != 1 {
1101                    panic!("Incorrect span returned {:?}", e.span());
1102                }
1103            }
1104        };
1105    }
1106
1107    #[test]
1108    fn test_multiline_lexeme() {
1109        let src = "%%
1110'.*' 'STR'
1111[ \\n] ;"
1112            .to_string();
1113        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1114        let mut map = HashMap::new();
1115        map.insert("STR", 0u8);
1116        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1117
1118        let lexer = lexerdef.lexer("'a\nb'\n");
1119        let lexemes = lexer
1120            .iter()
1121            .map(|x| x.unwrap())
1122            .collect::<Vec<DefaultLexeme<u8>>>();
1123        assert_eq!(lexemes.len(), 1);
1124        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 1), (2, 3)));
1125        assert_eq!(lexer.span_lines_str(lexemes[0].span()), "'a\nb'");
1126    }
1127
1128    #[test]
1129    fn test_token_span() {
1130        let src = "%%
1131a 'A'
1132b 'B'
1133[ \\n] ;"
1134            .to_string();
1135        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1136        assert_eq!(
1137            lexerdef.get_rule_by_name("A").unwrap().name_span(),
1138            Span::new(6, 7)
1139        );
1140        assert_eq!(
1141            lexerdef.get_rule_by_name("B").unwrap().name_span(),
1142            Span::new(12, 13)
1143        );
1144        let anonymous_rules = lexerdef
1145            .iter_rules()
1146            .filter(|rule| rule.name().is_none())
1147            .collect::<Vec<_>>();
1148        assert_eq!(anonymous_rules[0].name_span(), Span::new(21, 21));
1149    }
1150
1151    #[test]
1152    fn test_token_start_states() {
1153        let src = "%x EXCLUSIVE_START
1154%s INCLUSIVE_START
1155%%
1156a 'A'
1157b 'B'
1158[ \\n] ;"
1159            .to_string();
1160        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1161        assert_eq!(
1162            lexerdef.get_rule_by_name("A").unwrap().name_span(),
1163            Span::new(44, 45)
1164        );
1165        assert_eq!(
1166            lexerdef.get_rule_by_name("B").unwrap().name_span(),
1167            Span::new(50, 51)
1168        );
1169    }
1170
1171    #[test]
1172    fn test_rule_start_states() {
1173        let src = "%x EXCLUSIVE_START
1174%s INCLUSIVE_START
1175%%
1176<EXCLUSIVE_START>a 'A'
1177<INCLUSIVE_START>b 'B'
1178[ \\n] ;"
1179            .to_string();
1180        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1181        let a_rule = lexerdef.get_rule_by_name("A").unwrap();
1182        assert_eq!(a_rule.name_span(), Span::new(61, 62));
1183        assert_eq!(a_rule.re_str, "a");
1184
1185        let b_rule = lexerdef.get_rule_by_name("B").unwrap();
1186        assert_eq!(b_rule.name_span(), Span::new(84, 85));
1187        assert_eq!(b_rule.re_str, "b");
1188    }
1189
1190    #[test]
1191    fn test_state_matches_regular_no_rule_states() {
1192        let all_states = &[
1193            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1194            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1195        ];
1196        let rule_states = vec![];
1197        let current_state = &all_states[0];
1198        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1199            current_state,
1200            &rule_states,
1201        );
1202        assert!(m);
1203    }
1204
1205    #[test]
1206    fn test_state_matches_exclusive_no_rule_states() {
1207        let all_states = &[
1208            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1209            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1210        ];
1211        let rule_states = vec![];
1212        let current_state = &all_states[1];
1213        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1214            current_state,
1215            &rule_states,
1216        );
1217        assert!(!m);
1218    }
1219
1220    #[test]
1221    fn test_state_matches_regular_matching_rule_states() {
1222        let all_states = &[
1223            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1224            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1225        ];
1226        let rule_states = vec![0];
1227        let current_state = &all_states[0];
1228        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1229            current_state,
1230            &rule_states,
1231        );
1232        assert!(m);
1233    }
1234
1235    #[test]
1236    fn test_state_matches_exclusive_matching_rule_states() {
1237        let all_states = &[
1238            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1239            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1240        ];
1241        let rule_states = vec![1];
1242        let current_state = &all_states[1];
1243        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1244            current_state,
1245            &rule_states,
1246        );
1247        assert!(m);
1248    }
1249
1250    #[test]
1251    fn test_state_matches_regular_other_rule_states() {
1252        let all_states = &[
1253            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1254            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1255        ];
1256        let rule_states = vec![1];
1257        let current_state = &all_states[0];
1258        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1259            current_state,
1260            &rule_states,
1261        );
1262        assert!(!m);
1263    }
1264
1265    #[test]
1266    fn test_state_matches_exclusive_other_rule_states() {
1267        let all_states = &[
1268            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1269            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1270        ];
1271        let rule_states = vec![0];
1272        let current_state = &all_states[1];
1273        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1274            current_state,
1275            &rule_states,
1276        );
1277        assert!(!m);
1278    }
1279}
lrlex/lexer.rs

lrlex/
lexer.rs
