lrlex/
lexer.rs

1use std::{
2    collections::{HashMap, HashSet},
3    fmt::Debug,
4    hash::Hash,
5    marker::PhantomData,
6    slice::Iter,
7    str::FromStr,
8};
9
10use cfgrammar::{
11    NewlineCache, Span,
12    header::{GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Value},
13    span::Location,
14};
15use num_traits::{AsPrimitive, PrimInt, Unsigned};
16use regex::{Regex, RegexBuilder};
17
18use lrpar::{Lexeme, Lexer, LexerTypes, NonStreamingLexer};
19
20use crate::{
21    LRLexError, LexBuildError, LexBuildResult, StartStateId,
22    parser::{LexParser, StartState, StartStateOperation},
23};
24
25#[doc(hidden)]
26/// Corresponds to the options for `regex::RegexBuilder`.
27#[derive(Clone, Debug)]
28#[non_exhaustive]
29pub struct LexFlags {
30    // The following values when `None` grmtools provides default values for in `DEFAULT_LEX_FLAGS`
31    pub dot_matches_new_line: Option<bool>,
32    pub multi_line: Option<bool>,
33    pub octal: Option<bool>,
34    pub posix_escapes: Option<bool>,
35    pub allow_wholeline_comments: Option<bool>,
36
37    // All the following values when `None` default to the `regex` crate's default value.
38    pub case_insensitive: Option<bool>,
39    pub swap_greed: Option<bool>,
40    pub ignore_whitespace: Option<bool>,
41    pub unicode: Option<bool>,
42    pub size_limit: Option<usize>,
43    pub dfa_size_limit: Option<usize>,
44    pub nest_limit: Option<u32>,
45}
46
47impl<T: Clone> TryFrom<&mut Header<T>> for LexFlags {
48    type Error = HeaderError<T>;
49    fn try_from(header: &mut Header<T>) -> Result<LexFlags, HeaderError<T>> {
50        let mut lex_flags = UNSPECIFIED_LEX_FLAGS;
51        let LexFlags {
52            dot_matches_new_line,
53            multi_line,
54            octal,
55            posix_escapes,
56            allow_wholeline_comments,
57            case_insensitive,
58            swap_greed,
59            ignore_whitespace,
60            unicode,
61            size_limit,
62            dfa_size_limit,
63            nest_limit,
64        } = &mut lex_flags;
65        macro_rules! cvt_flag {
66            ($prefix:ident, $it:ident) => {
67                header.mark_used(&stringify!($prefix.$it).to_string());
68                *$it = match header.get(stringify!($prefix.$it)) {
69                    Some(HeaderValue(_, Value::Bool(flag, _))) => Some(*flag),
70                    Some(HeaderValue(_, val)) => Err(HeaderError {
71                        kind: HeaderErrorKind::ConversionError("LexFlags", "Expected boolean"),
72                        locations: vec![val.primary_location().clone()],
73                    })?,
74                    None => None,
75                }
76            };
77        }
78        cvt_flag!(regex, dot_matches_new_line);
79        cvt_flag!(regex, multi_line);
80        cvt_flag!(regex, octal);
81        cvt_flag!(lrlex, posix_escapes);
82        cvt_flag!(lrlex, allow_wholeline_comments);
83        cvt_flag!(regex, case_insensitive);
84        cvt_flag!(regex, swap_greed);
85        cvt_flag!(regex, ignore_whitespace);
86        cvt_flag!(regex, unicode);
87        macro_rules! cvt_num {
88            ($prefix:ident, $it:ident, $num_ty: ty) => {
89                header.mark_used(&stringify!($prefix.$it).to_string());
90                *$it = match header.get(stringify!($prefix.$it)) {
91                    Some(HeaderValue(_, Value::Num(n, _))) => Some(*n as $num_ty),
92                    Some(HeaderValue(_, val)) => Err(HeaderError {
93                        kind: HeaderErrorKind::ConversionError("LexFlags", "Expected numeric"),
94                        locations: vec![val.primary_location().clone()],
95                    })?,
96                    None => None,
97                }
98            };
99        }
100        cvt_num!(regex, size_limit, usize);
101        cvt_num!(regex, dfa_size_limit, usize);
102        cvt_num!(regex, nest_limit, u32);
103        Ok(lex_flags)
104    }
105}
106
107impl From<&LexFlags> for Header<Location> {
108    fn from(flags: &LexFlags) -> Header<Location> {
109        let mut header = Header::new();
110        let LexFlags {
111            dot_matches_new_line,
112            multi_line,
113            octal,
114            posix_escapes,
115            allow_wholeline_comments,
116            case_insensitive,
117            swap_greed,
118            ignore_whitespace,
119            unicode,
120            size_limit,
121            dfa_size_limit,
122            nest_limit,
123        } = flags;
124        macro_rules! cvt_flag {
125            ($it: ident) => {
126                $it.map(|x| {
127                    header.insert(
128                        stringify!($it).to_string(),
129                        HeaderValue(
130                            Location::Other("From<&LexFlags>".to_string()),
131                            Value::Bool(x, Location::Other("From<&LexFlags>".to_string())),
132                        ),
133                    )
134                });
135            };
136        }
137        cvt_flag!(dot_matches_new_line);
138        cvt_flag!(multi_line);
139        cvt_flag!(octal);
140        cvt_flag!(posix_escapes);
141        cvt_flag!(allow_wholeline_comments);
142        cvt_flag!(case_insensitive);
143        cvt_flag!(swap_greed);
144        cvt_flag!(ignore_whitespace);
145        cvt_flag!(unicode);
146
147        macro_rules! cvt_num {
148            ($it: ident) => {
149                $it.map(|x| {
150                    header.insert(
151                        stringify!($it).to_string(),
152                        HeaderValue(
153                            Location::Other("From<&LexFlags>".to_string()),
154                            Value::Num(x as u64, Location::Other("From<&LexFlags>".to_string())),
155                        ),
156                    )
157                });
158            };
159        }
160        cvt_num!(size_limit);
161        cvt_num!(dfa_size_limit);
162        cvt_num!(nest_limit);
163
164        header
165    }
166}
167
168/// LexFlags with flags set to default values.
169#[doc(hidden)]
170pub const DEFAULT_LEX_FLAGS: LexFlags = LexFlags {
171    allow_wholeline_comments: Some(false),
172    dot_matches_new_line: Some(true),
173    multi_line: Some(true),
174    octal: Some(true),
175    posix_escapes: Some(false),
176    case_insensitive: None,
177    ignore_whitespace: None,
178    swap_greed: None,
179    unicode: None,
180    size_limit: None,
181    dfa_size_limit: None,
182    nest_limit: None,
183};
184
185#[doc(hidden)]
186/// LexFlags with all of the values `None`.
187pub const UNSPECIFIED_LEX_FLAGS: LexFlags = LexFlags {
188    allow_wholeline_comments: None,
189    dot_matches_new_line: None,
190    multi_line: None,
191    octal: None,
192    posix_escapes: None,
193    case_insensitive: None,
194    ignore_whitespace: None,
195    swap_greed: None,
196    unicode: None,
197    size_limit: None,
198    dfa_size_limit: None,
199    nest_limit: None,
200};
201
202#[derive(Debug, Clone)]
203#[doc(hidden)]
204pub struct Rule<StorageT> {
205    /// If `Some`, this specifies the ID that lexemes resulting from this rule will have. Note that
206    /// lrlex gives rules a guaranteed unique value by default, though users can later override
207    /// that, potentially undermining uniqueness if they're not careful.
208    ///
209    /// If `None`, then this rule specifies lexemes which should not appear in the user's input.
210    pub(super) tok_id: Option<StorageT>,
211    /// This rule's name. If None, then text which matches this rule will be skipped (i.e. will not
212    /// create a lexeme).
213    #[deprecated(note = "Use the name() function")]
214    pub name: Option<String>,
215    #[deprecated(note = "Use the name_span() function")]
216    pub name_span: Span,
217    pub(super) re_str: String,
218    re: Regex,
219    /// Id(s) of permitted start conditions for the lexer to match this rule.
220    #[deprecated(note = "Use the start_states() function")]
221    pub start_states: Vec<usize>,
222    /// If Some(_), successful matching of this rule will cause the current stack of start
223    /// conditions in the lexer to be updated with the enclosed value, using the designated
224    /// operation.
225    /// If None, successful matching causes no change to the current start condition.
226    #[deprecated(note = "Use the target_state() function")]
227    pub target_state: Option<(usize, StartStateOperation)>,
228}
229
230impl<StorageT: PrimInt> Rule<StorageT> {
231    /// Create a new `Rule`. This interface is unstable and should only be used by code generated
232    /// by lrlex itself.
233    #[doc(hidden)]
234    #[allow(private_interfaces)]
235    #[allow(clippy::too_many_arguments)]
236    pub fn new(
237        _: crate::unstable_api::InternalPublicApi,
238        tok_id: Option<StorageT>,
239        name: Option<String>,
240        name_span: Span,
241        re_str: String,
242        start_states: Vec<usize>,
243        target_state: Option<(usize, StartStateOperation)>,
244        lex_flags: &LexFlags,
245    ) -> Result<Rule<StorageT>, regex::Error> {
246        let mut re = RegexBuilder::new(&format!("\\A(?:{})", re_str));
247        let mut re = re
248            .octal(lex_flags.octal.unwrap())
249            .multi_line(lex_flags.multi_line.unwrap())
250            .dot_matches_new_line(lex_flags.dot_matches_new_line.unwrap());
251
252        if let Some(flag) = lex_flags.ignore_whitespace {
253            re = re.ignore_whitespace(flag)
254        }
255        if let Some(flag) = lex_flags.unicode {
256            re = re.unicode(flag)
257        }
258        if let Some(flag) = lex_flags.case_insensitive {
259            re = re.case_insensitive(flag)
260        }
261        if let Some(flag) = lex_flags.swap_greed {
262            re = re.swap_greed(flag)
263        }
264        if let Some(sz) = lex_flags.size_limit {
265            re = re.size_limit(sz)
266        }
267        if let Some(sz) = lex_flags.dfa_size_limit {
268            re = re.dfa_size_limit(sz)
269        }
270        if let Some(lim) = lex_flags.nest_limit {
271            re = re.nest_limit(lim)
272        }
273
274        let re = re.build()?;
275        #[allow(deprecated)]
276        Ok(Rule {
277            tok_id,
278            name,
279            name_span,
280            re_str,
281            re,
282            start_states,
283            target_state,
284        })
285    }
286
287    /// Return this rule's token ID, if any.
288    ///
289    /// If `Some`, this specifies the ID that lexemes resulting from this rule will have. If
290    /// `None`, then this rule specifies lexemes which should not appear in the user's input.
291    pub fn tok_id(&self) -> Option<StorageT> {
292        self.tok_id
293    }
294
295    /// Return this rule's name. If `None`, then text which matches this rule will be skipped (i.e.
296    /// it will not result in the creation of a [Lexeme]).
297    pub fn name(&self) -> Option<&str> {
298        #[allow(deprecated)]
299        self.name.as_deref()
300    }
301
302    /// Return the [Span] of this rule's name.
303    pub fn name_span(&self) -> Span {
304        #[allow(deprecated)]
305        self.name_span
306    }
307
308    /// Return the original regular expression specified by the user for this [Rule].
309    pub fn re_str(&self) -> &str {
310        &self.re_str
311    }
312
313    /// Return the IDs of the permitted start conditions for the lexer to match this rule.
314    pub fn start_states(&self) -> &[usize] {
315        #[allow(deprecated)]
316        self.start_states.as_slice()
317    }
318
319    /// Return the IDs of the permitted start conditions for the lexer to match this rule.
320    pub fn target_state(&self) -> Option<(usize, StartStateOperation)> {
321        #[allow(deprecated)]
322        self.target_state.clone()
323    }
324}
325
326/// Methods which all lexer definitions must implement.
327pub trait LexerDef<LexerTypesT: LexerTypes>
328where
329    usize: AsPrimitive<LexerTypesT::StorageT>,
330{
331    #[doc(hidden)]
332    /// Instantiate a lexer from a set of `Rule`s. This is only intended to be used by compiled
333    /// lexers (see `ctbuilder.rs`).
334    fn from_rules(start_states: Vec<StartState>, rules: Vec<Rule<LexerTypesT::StorageT>>) -> Self
335    where
336        Self: Sized;
337
338    /// Instantiate a lexer from a string (e.g. representing a `.l` file).
339    fn from_str(s: &str) -> LexBuildResult<Self>
340    where
341        Self: Sized;
342
343    /// Get the `Rule` at index `idx`.
344    fn get_rule(&self, idx: usize) -> Option<&Rule<LexerTypesT::StorageT>>;
345
346    /// Get the `Rule` instance associated with a particular lexeme ID. Panics if no such rule
347    /// exists.
348    fn get_rule_by_id(&self, tok_id: LexerTypesT::StorageT) -> &Rule<LexerTypesT::StorageT>;
349
350    /// Get the `Rule` instance associated with a particular name.
351    fn get_rule_by_name(&self, n: &str) -> Option<&Rule<LexerTypesT::StorageT>>;
352
353    /// Set the id attribute on rules to the corresponding value in `map`. This is typically used
354    /// to synchronise a parser's notion of lexeme IDs with the lexers. While doing this, it keeps
355    /// track of which lexemes:
356    ///   1) are defined in the lexer but not referenced by the parser
357    ///   2) and referenced by the parser but not defined in the lexer
358    ///
359    /// and returns them as a tuple `(Option<HashSet<&str>>, Option<HashSet<&str>>)` in the order
360    /// (*defined_in_lexer_missing_from_parser*, *referenced_in_parser_missing_from_lexer*). Since
361    /// in most cases both sets are expected to be empty, `None` is returned to avoid a `HashSet`
362    /// allocation.
363    ///
364    /// Lexing and parsing can continue if either set is non-empty, so it is up to the caller as to
365    /// what action they take if either return set is non-empty. A non-empty set #1 is often
366    /// benign: some lexers deliberately define tokens which are not used (e.g. reserving future
367    /// keywords). A non-empty set #2 is more likely to be an error since there are parts of the
368    /// grammar where nothing the user can input will be parseable.
369    fn set_rule_ids<'a>(
370        &'a mut self,
371        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
372    ) -> (Option<HashSet<&'a str>>, Option<HashSet<&'a str>>);
373
374    fn set_rule_ids_spanned<'a>(
375        &'a mut self,
376        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
377    ) -> (Option<HashSet<&'a str>>, Option<HashSet<(&'a str, Span)>>);
378
379    /// Returns an iterator over all rules in this AST.
380    fn iter_rules(&self) -> Iter<'_, Rule<LexerTypesT::StorageT>>;
381
382    /// Returns an iterator over all start states in this AST.
383    fn iter_start_states(&self) -> Iter<'_, StartState>;
384}
385
386/// This struct represents, in essence, a .l file in memory. From it one can produce an
387/// [LRNonStreamingLexer] which actually lexes inputs.
388#[derive(Debug, Clone)]
389pub struct LRNonStreamingLexerDef<LexerTypesT: LexerTypes>
390where
391    usize: AsPrimitive<LexerTypesT::StorageT>,
392{
393    rules: Vec<Rule<LexerTypesT::StorageT>>,
394    start_states: Vec<StartState>,
395    lex_flags: LexFlags,
396    pub(crate) expected_missing_tokens: Vec<String>,
397    phantom: PhantomData<LexerTypesT>,
398}
399
400impl<LexerTypesT: LexerTypes> LexerDef<LexerTypesT> for LRNonStreamingLexerDef<LexerTypesT>
401where
402    usize: AsPrimitive<LexerTypesT::StorageT>,
403    LexerTypesT::StorageT: TryFrom<usize>,
404{
405    fn from_rules(
406        start_states: Vec<StartState>,
407        rules: Vec<Rule<LexerTypesT::StorageT>>,
408    ) -> LRNonStreamingLexerDef<LexerTypesT> {
409        LRNonStreamingLexerDef {
410            rules,
411            start_states,
412            lex_flags: DEFAULT_LEX_FLAGS,
413            expected_missing_tokens: vec![],
414            phantom: PhantomData,
415        }
416    }
417
418    /// Given a `.l` file in an `&str`, returns a `LrNonStreamingLexerDef`
419    /// after merging the `%grmtools` section with the default set of `LexFlags`.
420    fn from_str(s: &str) -> LexBuildResult<LRNonStreamingLexerDef<LexerTypesT>> {
421        let (mut header, pos) = GrmtoolsSectionParser::new(s, false)
422            .parse()
423            .map_err(|mut errs| errs.drain(..).map(LexBuildError::from).collect::<Vec<_>>())?;
424        let flags = LexFlags::try_from(&mut header).map_err(|e| vec![e.into()])?;
425        LexParser::<LexerTypesT>::new_with_lex_flags(s[pos..].to_string(), flags.clone()).map(|p| {
426            LRNonStreamingLexerDef {
427                rules: p.rules,
428                start_states: p.start_states,
429                lex_flags: flags,
430                expected_missing_tokens: p.expected_missing_tokens,
431                phantom: PhantomData,
432            }
433        })
434    }
435
436    fn get_rule(&self, idx: usize) -> Option<&Rule<LexerTypesT::StorageT>> {
437        self.rules.get(idx)
438    }
439
440    fn get_rule_by_id(&self, tok_id: LexerTypesT::StorageT) -> &Rule<LexerTypesT::StorageT> {
441        self.rules
442            .iter()
443            .find(|r| r.tok_id == Some(tok_id))
444            .unwrap()
445    }
446
447    fn get_rule_by_name(&self, n: &str) -> Option<&Rule<LexerTypesT::StorageT>> {
448        self.rules.iter().find(|r| r.name() == Some(n))
449    }
450
451    fn set_rule_ids<'a>(
452        &'a mut self,
453        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
454    ) -> (Option<HashSet<&'a str>>, Option<HashSet<&'a str>>) {
455        let (missing_from_parser, missing_from_lexer) = self.set_rule_ids_spanned(rule_ids_map);
456        let missing_from_lexer =
457            missing_from_lexer.map(|missing| missing.iter().map(|(name, _)| *name).collect());
458        (missing_from_parser, missing_from_lexer)
459    }
460
461    fn set_rule_ids_spanned<'a>(
462        &'a mut self,
463        rule_ids_map: &HashMap<&'a str, LexerTypesT::StorageT>,
464    ) -> (Option<HashSet<&'a str>>, Option<HashSet<(&'a str, Span)>>) {
465        // Because we have to iter_mut over self.rules, we can't easily store a reference to the
466        // rule's name at the same time. Instead, we store the index of each such rule and
467        // recover the names later. This has the unfortunate consequence of extended the mutable
468        // borrow for the rest of the 'a lifetime. To avoid that we could return idx's here.
469        // But the original `set_rule_ids` invalidates indexes.  In the spirit of keeping that
470        // behavior consistent, this also returns the span.
471        let mut missing_from_parser_idxs = Vec::new();
472        let mut rules_with_names = 0;
473        for (i, r) in self.rules.iter_mut().enumerate() {
474            if let Some(n) = r.name() {
475                match rule_ids_map.get(n) {
476                    Some(tok_id) => r.tok_id = Some(*tok_id),
477                    None => {
478                        r.tok_id = None;
479                        missing_from_parser_idxs.push(i);
480                    }
481                }
482                rules_with_names += 1;
483            }
484        }
485
486        let missing_from_parser = if missing_from_parser_idxs.is_empty() {
487            None
488        } else {
489            let mut mfp = HashSet::with_capacity(missing_from_parser_idxs.len());
490            for i in &missing_from_parser_idxs {
491                mfp.insert((self.rules[*i].name().unwrap(), self.rules[*i].name_span()));
492            }
493            Some(mfp)
494        };
495
496        let missing_from_lexer =
497            if rules_with_names - missing_from_parser_idxs.len() == rule_ids_map.len() {
498                None
499            } else {
500                Some(
501                    rule_ids_map
502                        .keys()
503                        .cloned()
504                        .collect::<HashSet<&str>>()
505                        .difference(
506                            &self
507                                .rules
508                                .iter()
509                                .filter_map(|x| x.name())
510                                .collect::<HashSet<&str>>(),
511                        )
512                        .cloned()
513                        .collect::<HashSet<&str>>(),
514                )
515            };
516
517        (missing_from_lexer, missing_from_parser)
518    }
519
520    fn iter_rules(&self) -> Iter<'_, Rule<LexerTypesT::StorageT>> {
521        self.rules.iter()
522    }
523
524    fn iter_start_states(&self) -> Iter<'_, StartState> {
525        self.start_states.iter()
526    }
527}
528
529impl<
530    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
531    LexerTypesT: LexerTypes<StorageT = StorageT>,
532> LRNonStreamingLexerDef<LexerTypesT>
533where
534    usize: AsPrimitive<StorageT>,
535    LexerTypesT::StorageT: TryFrom<usize>,
536{
537    /// Uses the `lex_flags` passed in ignoring any settings in the `%grmtools` section.
538    pub fn new_with_options(
539        s: &str,
540        lex_flags: LexFlags,
541    ) -> LexBuildResult<LRNonStreamingLexerDef<LexerTypesT>> {
542        let (_, pos) = GrmtoolsSectionParser::new(s, false).parse().unwrap();
543        LexParser::<LexerTypesT>::new_with_lex_flags(s[pos..].to_string(), lex_flags.clone()).map(
544            |p| LRNonStreamingLexerDef {
545                rules: p.rules,
546                start_states: p.start_states,
547                lex_flags,
548                expected_missing_tokens: p.expected_missing_tokens,
549                phantom: PhantomData,
550            },
551        )
552    }
553
554    /// Return an [LRNonStreamingLexer] for the `String` `s` that will lex relative to this
555    /// [LRNonStreamingLexerDef].
556    pub fn lexer<'lexer, 'input: 'lexer>(
557        &'lexer self,
558        s: &'input str,
559    ) -> LRNonStreamingLexer<'lexer, 'input, LexerTypesT> {
560        let mut lexemes = vec![];
561        let mut i = 0;
562        let mut state_stack: Vec<(usize, &StartState)> = Vec::new();
563        let initial_state = match self.get_start_state_by_id(0) {
564            None => {
565                lexemes.push(Err(LRLexError::new(Span::new(i, i))));
566                return LRNonStreamingLexer::new(s, lexemes, NewlineCache::from_str(s).unwrap());
567            }
568            Some(state) => state,
569        };
570        state_stack.push((1, initial_state));
571
572        while i < s.len() {
573            let old_i = i;
574            let mut longest = 0; // Length of the longest match
575            let mut longest_ridx = 0; // This is only valid iff longest != 0
576            let current_state = match state_stack.last() {
577                None => {
578                    lexemes.push(Err(LRLexError::new(Span::new(i, i))));
579                    return LRNonStreamingLexer::new(
580                        s,
581                        lexemes,
582                        NewlineCache::from_str(s).unwrap(),
583                    );
584                }
585                Some((_, s)) => s,
586            };
587            for (ridx, r) in self.iter_rules().enumerate() {
588                if !Self::state_matches(current_state, r.start_states()) {
589                    continue;
590                }
591                if let Some(m) = r.re.find(&s[old_i..]) {
592                    let len = m.end();
593                    // Note that by using ">", we implicitly prefer an earlier over a later rule, if
594                    // both match an input of the same length.
595                    if len > longest {
596                        longest = len;
597                        longest_ridx = ridx;
598                    }
599                }
600            }
601            if longest > 0 {
602                let r = self.get_rule(longest_ridx).unwrap();
603                if r.name().is_some() {
604                    match r.tok_id {
605                        Some(tok_id) => {
606                            lexemes.push(Ok(Lexeme::new(tok_id, old_i, longest)));
607                        }
608                        None => {
609                            lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
610                            break;
611                        }
612                    }
613                }
614                if let Some((target_state_id, op)) = &r.target_state() {
615                    let state = match self.get_start_state_by_id(*target_state_id) {
616                        None => {
617                            // TODO: I can see an argument for lexing state to be either `None` or `Some(target_state_id)` here
618                            lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
619                            break;
620                        }
621                        Some(state) => state,
622                    };
623                    let head = state_stack.last_mut();
624                    match op {
625                        StartStateOperation::ReplaceStack => {
626                            state_stack.clear();
627                            state_stack.push((1, state));
628                        }
629                        StartStateOperation::Push => match head {
630                            Some((count, s)) if s.id == state.id => *count += 1,
631                            _ => state_stack.push((1, state)),
632                        },
633                        StartStateOperation::Pop => match head {
634                            Some((count, _s)) if *count > 1 => {
635                                *count -= 1;
636                            }
637                            Some(_) => {
638                                state_stack.pop();
639                                if state_stack.is_empty() {
640                                    state_stack.push((1, initial_state));
641                                }
642                            }
643                            None => {
644                                lexemes.push(Err(LRLexError::new(Span::new(old_i, old_i))));
645                                break;
646                            }
647                        },
648                    }
649                }
650                i += longest;
651            } else {
652                lexemes.push(Err(LRLexError::new_with_lexing_state(
653                    Span::new(old_i, old_i),
654                    StartStateId::new(current_state.id),
655                )));
656                break;
657            }
658        }
659        LRNonStreamingLexer::new(s, lexemes, NewlineCache::from_str(s).unwrap())
660    }
661
662    fn state_matches(state: &StartState, rule_states: &[usize]) -> bool {
663        if rule_states.is_empty() {
664            !state.exclusive
665        } else {
666            rule_states.contains(&state.id)
667        }
668    }
669
670    fn get_start_state_by_id(&self, id: usize) -> Option<&StartState> {
671        self.start_states.iter().find(|state| state.id == id)
672    }
673
674    /// Returns the final `LexFlags` used for this lex source
675    /// after all forced and default flags have been resolved.
676    pub(crate) fn lex_flags(&self) -> Option<&LexFlags> {
677        Some(&self.lex_flags)
678    }
679}
680
681/// An `LRNonStreamingLexer` holds a reference to a string and can lex it into [lrpar::Lexeme]s.
682/// Although the struct is tied to a single string, no guarantees are made about whether the
683/// lexemes are cached or not.
684pub struct LRNonStreamingLexer<'lexer, 'input: 'lexer, LexerTypesT: LexerTypes>
685where
686    usize: AsPrimitive<LexerTypesT::StorageT>,
687    LexerTypesT::StorageT: 'static + Debug + PrimInt,
688{
689    s: &'input str,
690    lexemes: Vec<Result<LexerTypesT::LexemeT, LRLexError>>,
691    newlines: NewlineCache,
692    phantom: PhantomData<(&'lexer (), LexerTypesT::StorageT)>,
693}
694
695impl<
696    'lexer,
697    'input: 'lexer,
698    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
699    LexerTypesT: LexerTypes<StorageT = StorageT>,
700> LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
701where
702    usize: AsPrimitive<StorageT>,
703{
704    /// Create a new `LRNonStreamingLexer` that read in: the input `s`; and derived `lexemes` and
705    /// `newlines`.
706    ///
707    /// Note that if one or more lexemes or newlines was not created from `s`, subsequent calls to
708    /// the `LRNonStreamingLexer` may cause `panic`s.
709    pub fn new(
710        s: &'input str,
711        lexemes: Vec<Result<LexerTypesT::LexemeT, LRLexError>>,
712        newlines: NewlineCache,
713    ) -> LRNonStreamingLexer<'lexer, 'input, LexerTypesT> {
714        LRNonStreamingLexer {
715            s,
716            lexemes,
717            newlines,
718            phantom: PhantomData,
719        }
720    }
721}
722
723impl<
724    'lexer,
725    'input: 'lexer,
726    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
727    LexerTypesT: LexerTypes<StorageT = StorageT, LexErrorT = LRLexError>,
728> Lexer<LexerTypesT> for LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
729where
730    usize: AsPrimitive<StorageT>,
731{
732    fn iter<'a>(
733        &'a self,
734    ) -> Box<dyn Iterator<Item = Result<LexerTypesT::LexemeT, LexerTypesT::LexErrorT>> + 'a> {
735        Box::new(self.lexemes.iter().cloned())
736    }
737}
738
739impl<'lexer, 'input: 'lexer, LexerTypesT: LexerTypes<LexErrorT = LRLexError>>
740    NonStreamingLexer<'input, LexerTypesT> for LRNonStreamingLexer<'lexer, 'input, LexerTypesT>
741where
742    usize: AsPrimitive<LexerTypesT::StorageT>,
743{
744    fn span_str(&self, span: Span) -> &'input str {
745        if span.end() > self.s.len() {
746            panic!(
747                "Span {:?} exceeds known input length {}",
748                span,
749                self.s.len()
750            );
751        }
752        &self.s[span.start()..span.end()]
753    }
754
755    fn span_lines_str(&self, span: Span) -> &'input str {
756        debug_assert!(span.end() >= span.start());
757        if span.end() > self.s.len() {
758            panic!(
759                "Span {:?} exceeds known input length {}",
760                span,
761                self.s.len()
762            );
763        }
764
765        let (st, en) = self.newlines.span_line_bytes(span);
766        &self.s[st..en]
767    }
768
769    fn line_col(&self, span: Span) -> ((usize, usize), (usize, usize)) {
770        debug_assert!(span.end() >= span.start());
771        if span.end() > self.s.len() {
772            panic!(
773                "Span {:?} exceeds known input length {}",
774                span,
775                self.s.len()
776            );
777        }
778
779        (
780            self.newlines
781                .byte_to_line_num_and_col_num(self.s, span.start())
782                .unwrap(),
783            self.newlines
784                .byte_to_line_num_and_col_num(self.s, span.end())
785                .unwrap(),
786        )
787    }
788}
789
790#[cfg(test)]
791mod test {
792    use super::*;
793    use crate::{DefaultLexeme, DefaultLexerTypes};
794    use lrpar::LexError;
795    use std::collections::HashMap;
796
797    #[test]
798    fn test_basic() {
799        let src = r"
800%%
801[0-9]+ 'int'
802[a-zA-Z]+ 'id'
803[ \t] ;"
804            .to_string();
805        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
806        let mut map = HashMap::new();
807        map.insert("int", 0);
808        map.insert("id", 1);
809        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
810
811        let lexemes = lexerdef
812            .lexer("abc 123")
813            .iter()
814            .map(|x| x.unwrap())
815            .collect::<Vec<_>>();
816        assert_eq!(lexemes.len(), 2);
817        let lex1 = lexemes[0];
818        assert_eq!(lex1.tok_id(), 1u8);
819        assert_eq!(lex1.span().start(), 0);
820        assert_eq!(lex1.span().len(), 3);
821        let lex2 = lexemes[1];
822        assert_eq!(lex2.tok_id(), 0);
823        assert_eq!(lex2.span().start(), 4);
824        assert_eq!(lex2.span().len(), 3);
825    }
826
827    #[test]
828    fn test_posix_escapes() {
829        let src = r#"%%
830\\ 'slash'
831\a 'alert'
832\b 'backspace'
833\f 'feed'
834\n 'newline'
835\r 'return'
836\t 'tab'
837\v 'vtab'
838\q 'normal_char'
839"#
840        .to_string();
841        let mut options = DEFAULT_LEX_FLAGS;
842        options.posix_escapes = Some(true);
843        let lexerdef =
844            LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::new_with_options(&src, options)
845                .unwrap();
846        let lexemes = lexerdef
847            .lexer("\\\x07\x08\x0c\n\r\t\x0bq")
848            .iter()
849            .map(|x| x.unwrap())
850            .collect::<Vec<_>>();
851        assert_eq!(lexemes.len(), 9);
852        for i in 0..9u8 {
853            let lexeme = lexemes[i as usize];
854            assert_eq!(lexeme.tok_id(), i);
855        }
856    }
857
858    #[test]
859    fn test_non_posix_escapes() {
860        let src = r#"%%
861\\ 'slash'
862\a 'alert'
863a\b a 'work_break'
864\f 'feed'
865\n 'newline'
866\r 'return'
867\t 'tab'
868\v 'vtab'
869\q 'normal_char'
870"#
871        .to_string();
872        let mut options = DEFAULT_LEX_FLAGS;
873        options.posix_escapes = Some(false);
874        let lexerdef =
875            LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::new_with_options(&src, options)
876                .unwrap();
877        let lexemes = lexerdef
878            .lexer("\\\x07a a\x0c\n\r\t\x0bq")
879            .iter()
880            .map(|x| x.unwrap())
881            .collect::<Vec<_>>();
882        assert_eq!(lexemes.len(), 9);
883        for i in 0..9u8 {
884            let lexeme = lexemes[i as usize];
885            assert_eq!(lexeme.tok_id(), i);
886        }
887    }
888
889    #[test]
890    fn test_basic_error() {
891        let src = "
892%%
893[0-9]+ 'int'"
894            .to_string();
895        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
896        match lexerdef.lexer("abc").iter().next().unwrap() {
897            Ok(_) => panic!("Invalid input lexed"),
898            Err(e) => {
899                if e.span().start() != 0 || e.span().end() != 0 {
900                    panic!("Incorrect span returned {:?}", e.span());
901                }
902            }
903        };
904    }
905
906    #[test]
907    fn test_longest_match() {
908        let src = "%%
909if 'IF'
910[a-z]+ 'ID'
911[ ] ;"
912            .to_string();
913        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
914        let mut map = HashMap::new();
915        map.insert("IF", 0);
916        map.insert("ID", 1);
917        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
918
919        let lexemes = lexerdef
920            .lexer("iff if")
921            .iter()
922            .map(|x| x.unwrap())
923            .collect::<Vec<DefaultLexeme<u8>>>();
924        assert_eq!(lexemes.len(), 2);
925        let lex1 = lexemes[0];
926        assert_eq!(lex1.tok_id(), 1u8);
927        assert_eq!(lex1.span().start(), 0);
928        assert_eq!(lex1.span().len(), 3);
929        let lex2 = lexemes[1];
930        assert_eq!(lex2.tok_id(), 0);
931        assert_eq!(lex2.span().start(), 4);
932        assert_eq!(lex2.span().len(), 2);
933    }
934
935    #[test]
936    fn test_multibyte() {
937        let src = "%%
938[a❤]+ 'ID'
939[ ] ;"
940            .to_string();
941        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
942        let mut map = HashMap::new();
943        map.insert("ID", 0u8);
944        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
945
946        let lexer = lexerdef.lexer("a ❤ a");
947        let lexemes = lexer
948            .iter()
949            .map(|x| x.unwrap())
950            .collect::<Vec<DefaultLexeme<u8>>>();
951        assert_eq!(lexemes.len(), 3);
952        let lex1 = lexemes[0];
953        assert_eq!(lex1.span().start(), 0);
954        assert_eq!(lex1.span().len(), 1);
955        assert_eq!(lexer.span_str(lex1.span()), "a");
956        let lex2 = lexemes[1];
957        assert_eq!(lex2.span().start(), 2);
958        assert_eq!(lex2.span().len(), 3);
959        assert_eq!(lexer.span_str(lex2.span()), "❤");
960        let lex3 = lexemes[2];
961        assert_eq!(lex3.span().start(), 6);
962        assert_eq!(lex3.span().len(), 1);
963        assert_eq!(lexer.span_str(lex3.span()), "a");
964    }
965
966    #[test]
967    fn test_line_col() {
968        let src = "%%
969[a-z]+ 'ID'
970[ \\n] ;"
971            .to_string();
972        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
973        let mut map = HashMap::new();
974        map.insert("ID", 0u8);
975        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
976
977        let lexer = lexerdef.lexer("a b c");
978        let lexemes = lexer
979            .iter()
980            .map(|x| x.unwrap())
981            .collect::<Vec<DefaultLexeme<u8>>>();
982        assert_eq!(lexemes.len(), 3);
983        assert_eq!(lexer.line_col(lexemes[1].span()), ((1, 3), (1, 4)));
984        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a b c");
985        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "a b c");
986
987        let lexer = lexerdef.lexer("a b c\n");
988        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
989        assert_eq!(lexemes.len(), 3);
990        assert_eq!(lexer.line_col(lexemes[1].span()), ((1, 3), (1, 4)));
991        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a b c");
992        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "a b c");
993
994        let lexer = lexerdef.lexer(" a\nb\n  c d");
995        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
996        assert_eq!(lexemes.len(), 4);
997        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 2), (1, 3)));
998        assert_eq!(lexer.line_col(lexemes[1].span()), ((2, 1), (2, 2)));
999        assert_eq!(lexer.line_col(lexemes[2].span()), ((3, 3), (3, 4)));
1000        assert_eq!(lexer.line_col(lexemes[3].span()), ((3, 5), (3, 6)));
1001        assert_eq!(lexer.span_lines_str(lexemes[0].span()), " a");
1002        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "b");
1003        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "  c d");
1004        assert_eq!(lexer.span_lines_str(lexemes[3].span()), "  c d");
1005
1006        let mut s = Vec::new();
1007        let mut offs = vec![0];
1008        for i in 0..71 {
1009            offs.push(offs[i] + i + 1);
1010            s.push(vec!["a"; i].join(" "));
1011        }
1012        let s = s.join("\n");
1013        let lexer = lexerdef.lexer(&s);
1014        let lexemes = lexer.iter().map(|x| x.unwrap()).collect::<Vec<_>>();
1015        assert_eq!(lexemes.len(), offs[70]);
1016        assert_eq!(lexer.span_lines_str(Span::new(0, 0)), "");
1017        assert_eq!(lexer.span_lines_str(Span::new(0, 2)), "\na");
1018        assert_eq!(lexer.span_lines_str(Span::new(0, 4)), "\na\na a");
1019        assert_eq!(lexer.span_lines_str(Span::new(0, 7)), "\na\na a\na a a");
1020        assert_eq!(lexer.span_lines_str(Span::new(4, 7)), "a a\na a a");
1021        assert_eq!(lexer.span_lines_str(lexemes[0].span()), "a");
1022        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "a a");
1023        assert_eq!(lexer.span_lines_str(lexemes[3].span()), "a a a");
1024        for i in 0..70 {
1025            assert_eq!(
1026                lexer.span_lines_str(lexemes[offs[i]].span()),
1027                vec!["a"; i + 1].join(" ")
1028            );
1029        }
1030    }
1031
1032    #[test]
1033    fn test_line_col_multibyte() {
1034        let src = "%%
1035[a-z❤]+ 'ID'
1036[ \\n] ;"
1037            .to_string();
1038        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1039        let mut map = HashMap::new();
1040        map.insert("ID", 0u8);
1041        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1042
1043        let lexer = lexerdef.lexer(" a\n❤ b");
1044        let lexemes = lexer
1045            .iter()
1046            .map(|x| x.unwrap())
1047            .collect::<Vec<DefaultLexeme<u8>>>();
1048        assert_eq!(lexemes.len(), 3);
1049        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 2), (1, 3)));
1050        assert_eq!(lexer.line_col(lexemes[1].span()), ((2, 1), (2, 2)));
1051        assert_eq!(lexer.line_col(lexemes[2].span()), ((2, 3), (2, 4)));
1052        assert_eq!(lexer.span_lines_str(lexemes[0].span()), " a");
1053        assert_eq!(lexer.span_lines_str(lexemes[1].span()), "❤ b");
1054        assert_eq!(lexer.span_lines_str(lexemes[2].span()), "❤ b");
1055    }
1056
1057    #[test]
1058    #[should_panic]
1059    fn test_bad_line_col() {
1060        let src = "%%
1061[a-z]+ 'ID'
1062[ \\n] ;"
1063            .to_string();
1064        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1065        let mut map = HashMap::new();
1066        map.insert("ID", 0u8);
1067        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1068
1069        let lexer = lexerdef.lexer("a b c");
1070
1071        lexer.line_col(Span::new(100, 100));
1072    }
1073
1074    #[test]
1075    fn test_missing_from_lexer_and_parser() {
1076        let src = "%%
1077[a-z]+ 'ID'
1078[ \\n] ;"
1079            .to_string();
1080        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1081        let mut map = HashMap::new();
1082        map.insert("INT", 0u8);
1083        let mut missing_from_lexer = HashSet::new();
1084        missing_from_lexer.insert("INT");
1085        let mut missing_from_parser = HashSet::new();
1086        missing_from_parser.insert("ID");
1087        assert_eq!(
1088            lexerdef.set_rule_ids(&map),
1089            (Some(missing_from_lexer), Some(missing_from_parser))
1090        );
1091
1092        match lexerdef.lexer(" a ").iter().next().unwrap() {
1093            Ok(_) => panic!("Invalid input lexed"),
1094            Err(e) => {
1095                if e.span().start() != 1 || e.span().end() != 1 {
1096                    panic!("Incorrect span returned {:?}", e.span());
1097                }
1098            }
1099        };
1100    }
1101
1102    #[test]
1103    fn test_multiline_lexeme() {
1104        let src = "%%
1105'.*' 'STR'
1106[ \\n] ;"
1107            .to_string();
1108        let mut lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1109        let mut map = HashMap::new();
1110        map.insert("STR", 0u8);
1111        assert_eq!(lexerdef.set_rule_ids(&map), (None, None));
1112
1113        let lexer = lexerdef.lexer("'a\nb'\n");
1114        let lexemes = lexer
1115            .iter()
1116            .map(|x| x.unwrap())
1117            .collect::<Vec<DefaultLexeme<u8>>>();
1118        assert_eq!(lexemes.len(), 1);
1119        assert_eq!(lexer.line_col(lexemes[0].span()), ((1, 1), (2, 3)));
1120        assert_eq!(lexer.span_lines_str(lexemes[0].span()), "'a\nb'");
1121    }
1122
1123    #[test]
1124    fn test_token_span() {
1125        let src = "%%
1126a 'A'
1127b 'B'
1128[ \\n] ;"
1129            .to_string();
1130        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1131        assert_eq!(
1132            lexerdef.get_rule_by_name("A").unwrap().name_span(),
1133            Span::new(6, 7)
1134        );
1135        assert_eq!(
1136            lexerdef.get_rule_by_name("B").unwrap().name_span(),
1137            Span::new(12, 13)
1138        );
1139        let anonymous_rules = lexerdef
1140            .iter_rules()
1141            .filter(|rule| rule.name().is_none())
1142            .collect::<Vec<_>>();
1143        assert_eq!(anonymous_rules[0].name_span(), Span::new(21, 21));
1144    }
1145
1146    #[test]
1147    fn test_token_start_states() {
1148        let src = "%x EXCLUSIVE_START
1149%s INCLUSIVE_START
1150%%
1151a 'A'
1152b 'B'
1153[ \\n] ;"
1154            .to_string();
1155        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1156        assert_eq!(
1157            lexerdef.get_rule_by_name("A").unwrap().name_span(),
1158            Span::new(44, 45)
1159        );
1160        assert_eq!(
1161            lexerdef.get_rule_by_name("B").unwrap().name_span(),
1162            Span::new(50, 51)
1163        );
1164    }
1165
1166    #[test]
1167    fn test_rule_start_states() {
1168        let src = "%x EXCLUSIVE_START
1169%s INCLUSIVE_START
1170%%
1171<EXCLUSIVE_START>a 'A'
1172<INCLUSIVE_START>b 'B'
1173[ \\n] ;"
1174            .to_string();
1175        let lexerdef = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(&src).unwrap();
1176        let a_rule = lexerdef.get_rule_by_name("A").unwrap();
1177        assert_eq!(a_rule.name_span(), Span::new(61, 62));
1178        assert_eq!(a_rule.re_str, "a");
1179
1180        let b_rule = lexerdef.get_rule_by_name("B").unwrap();
1181        assert_eq!(b_rule.name_span(), Span::new(84, 85));
1182        assert_eq!(b_rule.re_str, "b");
1183    }
1184
1185    #[test]
1186    fn test_state_matches_regular_no_rule_states() {
1187        let all_states = &[
1188            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1189            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1190        ];
1191        let rule_states = vec![];
1192        let current_state = &all_states[0];
1193        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1194            current_state,
1195            &rule_states,
1196        );
1197        assert!(m);
1198    }
1199
1200    #[test]
1201    fn test_state_matches_exclusive_no_rule_states() {
1202        let all_states = &[
1203            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1204            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1205        ];
1206        let rule_states = vec![];
1207        let current_state = &all_states[1];
1208        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1209            current_state,
1210            &rule_states,
1211        );
1212        assert!(!m);
1213    }
1214
1215    #[test]
1216    fn test_state_matches_regular_matching_rule_states() {
1217        let all_states = &[
1218            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1219            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1220        ];
1221        let rule_states = vec![0];
1222        let current_state = &all_states[0];
1223        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1224            current_state,
1225            &rule_states,
1226        );
1227        assert!(m);
1228    }
1229
1230    #[test]
1231    fn test_state_matches_exclusive_matching_rule_states() {
1232        let all_states = &[
1233            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1234            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1235        ];
1236        let rule_states = vec![1];
1237        let current_state = &all_states[1];
1238        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1239            current_state,
1240            &rule_states,
1241        );
1242        assert!(m);
1243    }
1244
1245    #[test]
1246    fn test_state_matches_regular_other_rule_states() {
1247        let all_states = &[
1248            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1249            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1250        ];
1251        let rule_states = vec![1];
1252        let current_state = &all_states[0];
1253        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1254            current_state,
1255            &rule_states,
1256        );
1257        assert!(!m);
1258    }
1259
1260    #[test]
1261    fn test_state_matches_exclusive_other_rule_states() {
1262        let all_states = &[
1263            StartState::new(0, "INITIAL", false, Span::new(0, 0)),
1264            StartState::new(1, "EXCLUSIVE", true, Span::new(0, 0)),
1265        ];
1266        let rule_states = vec![0];
1267        let current_state = &all_states[1];
1268        let m = LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::state_matches(
1269            current_state,
1270            &rule_states,
1271        );
1272        assert!(!m);
1273    }
1274}
lrlex/lexer.rs

lrlex/
lexer.rs
