lrlex/
ctbuilder.rs

1//! Build grammars at run-time.
2
3use cfgrammar::{
4    header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value},
5    markmap::MergeBehavior,
6    span::{Location, Span},
7};
8use glob::glob;
9use lrpar::{
10    CTParserBuilder, LexerTypes,
11    diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter},
12};
13use num_traits::{AsPrimitive, PrimInt, Unsigned};
14use proc_macro2::{Ident, TokenStream};
15use quote::{ToTokens, format_ident, quote};
16use std::{
17    any::type_name,
18    borrow::Borrow,
19    collections::{HashMap, HashSet},
20    env::{current_dir, var},
21    error::Error,
22    fmt::{self, Debug, Display, Write as _},
23    fs::{self, File, create_dir_all, read_to_string},
24    hash::Hash,
25    io::Write,
26    marker::PhantomData,
27    path::{Path, PathBuf},
28    sync::{LazyLock, Mutex},
29};
30use wincode::SchemaWrite;
31
32use crate::{
33    DefaultLexerTypes, LRNonStreamingLexer, LexerDef,
34    codegen::{LexerBuildEnvArgs, LexerSrcEnv, LexerSrcEnvError},
35};
36
37const RUST_FILE_EXT: &str = "rs";
38
39const ERROR: &str = "[Error]";
40const WARNING: &str = "[Warning]";
41
42static GENERATED_PATHS: LazyLock<Mutex<HashSet<PathBuf>>> =
43    LazyLock::new(|| Mutex::new(HashSet::new()));
44
45#[non_exhaustive]
46pub enum LexerKind {
47    LRNonStreamingLexer,
48}
49
50impl<T: Clone> TryFrom<&Value<T>> for LexerKind {
51    type Error = cfgrammar::header::HeaderError<T>;
52    fn try_from(it: &Value<T>) -> Result<LexerKind, Self::Error> {
53        match it {
54            Value::Flag(_, loc) => Err(HeaderError {
55                kind: HeaderErrorKind::ConversionError(
56                    "LexerKind",
57                    "Expected `LexerKind` found bool",
58                ),
59                locations: vec![loc.clone()],
60            }),
61            Value::Setting(Setting::Num(_, loc)) => Err(HeaderError {
62                kind: HeaderErrorKind::ConversionError(
63                    "LexerKind",
64                    "Expected `LexerKind` found numeric",
65                ),
66                locations: vec![loc.clone()],
67            }),
68            Value::Setting(Setting::String(_, loc)) => Err(HeaderError {
69                kind: HeaderErrorKind::ConversionError(
70                    "LexerKind",
71                    "Expected `LexerKind` found string",
72                ),
73                locations: vec![loc.clone()],
74            }),
75            Value::Setting(Setting::Constructor {
76                ctor:
77                    Namespaced {
78                        namespace: _,
79                        member: (_, loc),
80                    },
81                arg: _,
82            }) => Err(HeaderError {
83                kind: HeaderErrorKind::ConversionError(
84                    "LexerKind",
85                    "Expected `LexerKind` found constructor",
86                ),
87                locations: vec![loc.clone()],
88            }),
89            Value::Setting(Setting::Array(_, arr_loc, _)) => Err(HeaderError {
90                kind: HeaderErrorKind::ConversionError(
91                    "LexerKind",
92                    "Expected `LexerKind` found array",
93                ),
94                locations: vec![arr_loc.clone()],
95            }),
96            Value::Setting(Setting::Unitary(Namespaced {
97                namespace,
98                member: (member, member_loc),
99            })) => {
100                if let Some((ns, loc)) = namespace
101                    && ns.to_lowercase() != "lexerkind"
102                {
103                    return Err(HeaderError {
104                        kind: HeaderErrorKind::ConversionError(
105                            "LexerKind",
106                            "Expected namespace `LexerKind`",
107                        ),
108                        locations: vec![loc.clone()],
109                    });
110                }
111                if member.to_lowercase() != "lrnonstreaminglexer" {
112                    return Err(HeaderError {
113                        kind: HeaderErrorKind::ConversionError(
114                            "LexerKind",
115                            "Unknown `LexerKind` Variant",
116                        ),
117                        locations: vec![member_loc.clone()],
118                    });
119                }
120
121                Ok(LexerKind::LRNonStreamingLexer)
122            }
123        }
124    }
125}
126
127/// Specify the visibility of the module generated by [CTLexerBuilder].
128#[derive(Clone, PartialEq, Eq, Debug)]
129#[non_exhaustive]
130pub enum Visibility {
131    /// Module-level visibility only.
132    Private,
133    /// `pub`
134    Public,
135    /// `pub(super)`
136    PublicSuper,
137    /// `pub(self)`
138    PublicSelf,
139    /// `pub(crate)`
140    PublicCrate,
141    /// `pub(in {arg})`
142    PublicIn(String),
143}
144
145impl ToTokens for Visibility {
146    fn to_tokens(&self, tokens: &mut TokenStream) {
147        tokens.extend(match self {
148            Visibility::Private => quote!(),
149            Visibility::Public => quote! {pub},
150            Visibility::PublicSuper => quote! {pub(super)},
151            Visibility::PublicSelf => quote! {pub(self)},
152            Visibility::PublicCrate => quote! {pub(crate)},
153            Visibility::PublicIn(data) => {
154                let other = str::parse::<TokenStream>(data).unwrap();
155                quote! {pub(in #other)}
156            }
157        })
158    }
159}
160
161/// Specifies the [Rust Edition] that will be emitted during code generation.
162///
163/// [Rust Edition]: https://doc.rust-lang.org/edition-guide/rust-2021/index.html
164#[derive(Clone, Copy, PartialEq, Eq, Debug)]
165#[non_exhaustive]
166pub enum RustEdition {
167    Rust2015,
168    Rust2018,
169    Rust2021,
170}
171
172/// A string which uses `Display` for it's `Debug` impl.
173struct ErrorString(String);
174impl fmt::Display for ErrorString {
175    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
176        let ErrorString(s) = self;
177        write!(f, "{}", s)
178    }
179}
180impl fmt::Debug for ErrorString {
181    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
182        let ErrorString(s) = self;
183        write!(f, "{}", s)
184    }
185}
186impl Error for ErrorString {}
187
188/// A `CTLexerBuilder` allows one to specify the criteria for building a statically generated
189/// lexer.
190pub struct CTLexerBuilder<'a, LexerTypesT: LexerTypes = DefaultLexerTypes<u32>>
191where
192    LexerTypesT::StorageT: Debug + Eq + Hash + ToTokens,
193    usize: num_traits::AsPrimitive<LexerTypesT::StorageT>,
194{
195    lrpar_config:
196        Option<Box<dyn Fn(CTParserBuilder<LexerTypesT>) -> CTParserBuilder<LexerTypesT> + 'a>>,
197    lexer_path: Option<PathBuf>,
198    output_path: Option<PathBuf>,
199    lexerkind: Option<LexerKind>,
200    mod_name: Option<&'a str>,
201    visibility: Visibility,
202    rust_edition: RustEdition,
203    rule_ids_map: Option<HashMap<String, LexerTypesT::StorageT>>,
204    allow_missing_terms_in_lexer: bool,
205    allow_missing_tokens_in_parser: bool,
206    warnings_are_errors: bool,
207    show_warnings: bool,
208    header: Header<Location>,
209    #[cfg(test)]
210    inspect_lexerkind_cb: Option<Box<dyn Fn(&LexerKind) -> Result<(), Box<dyn Error>>>>,
211}
212
213impl CTLexerBuilder<'_, DefaultLexerTypes<u32>> {
214    /// Create a new [CTLexerBuilder].
215    pub fn new() -> Self {
216        CTLexerBuilder::<DefaultLexerTypes<u32>>::new_with_lexemet()
217    }
218}
219type FixIntConfig = wincode::config::Configuration;
220
221type VarIntConfig = wincode::config::Configuration<
222    true,
223    4194304,
224    wincode::len::BincodeLen,
225    wincode::int_encoding::LittleEndian,
226    wincode::int_encoding::VarInt,
227>;
228impl<'a, LexerTypesT: LexerTypes<LexErrorT = crate::LRLexError> + 'static>
229    CTLexerBuilder<'a, LexerTypesT>
230where
231    LexerTypesT::StorageT: 'static
232        + Debug
233        + Eq
234        + Hash
235        + PrimInt
236        + SchemaWrite<FixIntConfig, Src = LexerTypesT::StorageT>
237        + SchemaWrite<VarIntConfig, Src = LexerTypesT::StorageT>
238        + TryFrom<usize>
239        + Unsigned
240        + ToTokens,
241    usize: AsPrimitive<LexerTypesT::StorageT>,
242{
243    /// Create a new [CTLexerBuilder].
244    ///
245    /// `LexerTypesT::StorageT` must be an unsigned integer type (e.g. `u8`, `u16`) which is big enough
246    /// to index all the tokens, rules, and productions in the lexer and less than or equal in size
247    /// to `usize` (e.g. on a 64-bit machine `u128` would be too big). If you are lexing large
248    /// files, the additional storage requirements of larger integer types can be noticeable, and
249    /// in such cases it can be worth specifying a smaller type. `StorageT` defaults to `u32` if
250    /// unspecified.
251    ///
252    /// # Examples
253    ///
254    /// ```text
255    /// CTLexerBuilder::<DefaultLexerTypes<u8>>::new_with_lexemet()
256    ///     .lexer_in_src_dir("grm.l", None)?
257    ///     .build()?;
258    /// ```
259    pub fn new_with_lexemet() -> Self {
260        let mut header = Header::new();
261        header.set_default_merge_behavior(MergeBehavior::Ours);
262        CTLexerBuilder {
263            lrpar_config: None,
264            lexer_path: None,
265            output_path: None,
266            lexerkind: None,
267            mod_name: None,
268            visibility: Visibility::Private,
269            rust_edition: RustEdition::Rust2021,
270            rule_ids_map: None,
271            allow_missing_terms_in_lexer: false,
272            allow_missing_tokens_in_parser: false,
273            warnings_are_errors: false,
274            show_warnings: true,
275            header,
276            #[cfg(test)]
277            inspect_lexerkind_cb: None,
278        }
279    }
280
281    /// An optional convenience function to make it easier to create an (lrlex) lexer and (lrpar)
282    /// parser in one shot. The closure passed to this function will be called during
283    /// [CTLexerBuilder::build]: it will be passed an lrpar `CTParserBuilder` instance upon which
284    /// it can set whatever lrpar options are desired. [`CTLexerBuilder`] will then create both the
285    /// compiler and lexer and link them together as required.
286    ///
287    /// # Examples
288    ///
289    /// ```text
290    /// CTLexerBuilder:::new()
291    ///     .lrpar_config(|ctp| {
292    ///         ctp.yacckind(YaccKind::Grmtools)
293    ///             .grammar_in_src_dir("calc.y")
294    ///             .unwrap()
295    ///     })
296    ///     .lexer_in_src_dir("calc.l")?
297    ///     .build()?;
298    /// ```
299    pub fn lrpar_config<F>(mut self, config_func: F) -> Self
300    where
301        F: Fn(CTParserBuilder<LexerTypesT>) -> CTParserBuilder<LexerTypesT> + 'a,
302    {
303        self.lrpar_config = Some(Box::new(config_func));
304        self
305    }
306
307    /// Set the input lexer path to a file relative to this project's `src` directory. This will
308    /// also set the output path (i.e. you do not need to call [CTLexerBuilder::output_path]).
309    ///
310    /// For example if `a/b.l` is passed as `inp` then [CTLexerBuilder::build] will:
311    ///   * use `src/a/b.l` as the input file.
312    ///   * write output to a file which can then be imported by calling `lrlex_mod!("a/b.l")`.
313    ///   * create a module in that output file named `b_l`.
314    ///
315    /// You can override the output path and/or module name by calling
316    /// [CTLexerBuilder::output_path] and/or [CTLexerBuilder::mod_name], respectively, after
317    /// calling this function.
318    ///
319    /// This is a convenience function that makes it easier to compile lexer files stored in a
320    /// project's `src/` directory: please see [CTLexerBuilder::build] for additional constraints
321    /// and information about the generated files. Note also that each `.l` file can only be
322    /// processed once using this function: if you want to generate multiple lexers from a single
323    /// `.l` file, you will need to use [CTLexerBuilder::output_path].
324    pub fn lexer_in_src_dir<P>(mut self, srcp: P) -> Result<Self, Box<dyn Error>>
325    where
326        P: AsRef<Path>,
327    {
328        if !srcp.as_ref().is_relative() {
329            return Err(format!(
330                "Lexer path '{}' must be a relative path.",
331                srcp.as_ref().to_str().unwrap_or("<invalid UTF-8>")
332            )
333            .into());
334        }
335
336        let mut lexp = current_dir()?;
337        lexp.push("src");
338        lexp.push(srcp.as_ref());
339        self.lexer_path = Some(lexp);
340
341        let mut outp = PathBuf::new();
342        outp.push(var("OUT_DIR").unwrap());
343        outp.push(srcp.as_ref().parent().unwrap().to_str().unwrap());
344        create_dir_all(&outp)?;
345        let mut leaf = srcp
346            .as_ref()
347            .file_name()
348            .unwrap()
349            .to_str()
350            .unwrap()
351            .to_owned();
352        write!(leaf, ".{}", RUST_FILE_EXT).ok();
353        outp.push(leaf);
354        Ok(self.output_path(outp))
355    }
356
357    /// Set the input lexer path to `inp`. If specified, you must also call
358    /// [CTLexerBuilder::output_path]. In general it is easier to use
359    /// [CTLexerBuilder::lexer_in_src_dir].
360    pub fn lexer_path<P>(mut self, inp: P) -> Self
361    where
362        P: AsRef<Path>,
363    {
364        self.lexer_path = Some(inp.as_ref().to_owned());
365        self
366    }
367
368    /// Set the output lexer path to `outp`. Note that there are no requirements on `outp`: the
369    /// file can exist anywhere you can create a valid [Path] to. However, if you wish to use
370    /// [crate::lrlex_mod!] you will need to make sure that `outp` is in
371    /// [std::env::var]`("OUT_DIR")` or one of its subdirectories.
372    pub fn output_path<P>(mut self, outp: P) -> Self
373    where
374        P: AsRef<Path>,
375    {
376        self.output_path = Some(outp.as_ref().to_owned());
377        self
378    }
379
380    /// Set the type of lexer to be generated to `lexerkind`.
381    pub fn lexerkind(mut self, lexerkind: LexerKind) -> Self {
382        self.lexerkind = Some(lexerkind);
383        self
384    }
385
386    /// Set the generated module name to `mod_name`. If no module name is specified,
387    /// [`process_file`](#method.process_file) will attempt to create a sensible default based on
388    /// the input filename.
389    pub fn mod_name(mut self, mod_name: &'a str) -> Self {
390        self.mod_name = Some(mod_name);
391        self
392    }
393
394    /// Set the visibility of the generated module to `vis`. Defaults to `Visibility::Private`.
395    pub fn visibility(mut self, vis: Visibility) -> Self {
396        self.visibility = vis;
397        self
398    }
399
400    /// Sets the rust edition to be used for generated code. Defaults to the latest edition of
401    /// rust supported by grmtools.
402    pub fn rust_edition(mut self, edition: RustEdition) -> Self {
403        self.rust_edition = edition;
404        self
405    }
406
407    /// Set this lexer builder's map of rule IDs to `rule_ids_map`. By default, lexing rules have
408    /// arbitrary, but distinct, IDs. Setting the map of rule IDs (from rule names to `StorageT`)
409    /// allows users to synchronise a lexer and parser and to check that all rules are used by both
410    /// parts).
411    pub fn rule_ids_map<T: std::borrow::Borrow<HashMap<String, LexerTypesT::StorageT>> + Clone>(
412        mut self,
413        rule_ids_map: T,
414    ) -> Self {
415        self.rule_ids_map = Some(rule_ids_map.borrow().to_owned());
416        self
417    }
418
419    /// Statically compile the `.l` file specified by [CTLexerBuilder::lexer_path()] into Rust,
420    /// placing the output into the file specified by [CTLexerBuilder::output_path()].
421    ///
422    /// The generated module follows the form:
423    ///
424    /// ```text
425    ///    mod modname {
426    ///      pub fn lexerdef() -> LexerDef<LexerTypesT> { ... }
427    ///
428    ///      ...
429    ///    }
430    /// ```
431    ///
432    /// where:
433    ///  * `modname` is either:
434    ///    * the module name specified by [CTLexerBuilder::mod_name()]
435    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.l` the
436    ///      module name is `c_l` (i.e. the file's leaf name, minus its extension, with a prefix of
437    ///      `_l`).
438    pub fn build(mut self) -> Result<CTLexer, Box<dyn Error>> {
439        let lexerp = self
440            .lexer_path
441            .as_ref()
442            .expect("lexer_path must be specified before processing.");
443        let outp = self
444            .output_path
445            .as_ref()
446            .expect("output_path must be specified before processing.");
447
448        {
449            let mut lk = GENERATED_PATHS.lock().unwrap();
450            if lk.contains(outp.as_path()) {
451                return Err(format!("Generating two lexers to the same path ('{}') is not allowed: use CTLexerBuilder::output_path (and, optionally, CTLexerBuilder::mod_name) to differentiate them.", outp.to_str().unwrap()).into());
452            }
453            lk.insert(outp.clone());
454        }
455        let lex_src = read_to_string(lexerp)
456            .map_err(|e| format!("When reading '{}': {e}", lexerp.display()))?;
457        let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, lexerp);
458        let args = LexerBuildEnvArgs::new()
459            .mod_name(self.mod_name.map(|s| s.to_string()))
460            .lexerkind(self.lexerkind)
461            .visibility(self.visibility);
462        let mut build_env =
463            LexerSrcEnv::<LexerTypesT>::new_with_header(&lex_src, Some(lexerp), self.header)
464                .build_env(args)
465                .map_err(|e| match e {
466                    LexerSrcEnvError::GrmtoolsSectionParseError(es) => {
467                        let mut out = String::new();
468                        out.push_str(&format!(
469                            "\n{ERROR}{}\n",
470                            lex_diag.file_location_msg(" parsing the `%grmtools` section", None)
471                        ));
472                        for e in es {
473                            out.push_str(&indent("     ", &lex_diag.format_error(e).to_string()));
474                            out.push('\n');
475                        }
476                        ErrorString(out)
477                    }
478                    LexerSrcEnvError::LexBuildErrors(errs) => {
479                        let mut out = String::new();
480                        out.push_str(&format!(
481                            "\n{ERROR}{}\n",
482                            lex_diag.file_location_msg(" building the lexer", None)
483                        ));
484                        for e in errs {
485                            out.push_str(&indent("     ", &lex_diag.format_error(e).to_string()));
486                            out.push('\n');
487                        }
488                        ErrorString(out)
489                    }
490                    e => ErrorString(e.to_string()),
491                })?;
492
493        #[cfg(test)]
494        if let Some(inspect_lexerkind_cb) = self.inspect_lexerkind_cb {
495            inspect_lexerkind_cb(build_env.lexerkind())?
496        }
497
498        let ct_parser = if let Some(ref lrcfg) = self.lrpar_config {
499            let mut closure_lexerdef = build_env.lexerdef().clone();
500            let mut ctp = CTParserBuilder::<LexerTypesT>::new().inspect_rt(Box::new(
501                move |yacc_header, rtpb, rule_ids_map, grm_path| {
502                    let owned_map = rule_ids_map
503                        .iter()
504                        .map(|(x, y)| (&**x, *y))
505                        .collect::<HashMap<_, _>>();
506                    closure_lexerdef.set_rule_ids(&owned_map);
507                    yacc_header.mark_used(&"lrpar.test_files".to_string());
508                    let grammar = rtpb.grammar();
509                    let test_glob = yacc_header.get("lrpar.test_files");
510                    let mut err_str = None;
511                    let add_error_line = |err_str: &mut Option<String>, line| {
512                        if let Some(err_str) = err_str {
513                            err_str.push_str(&format!("{}\n", line));
514                        } else {
515                            let _ = err_str.insert(format!("{}\n", line));
516                        }
517                    };
518                    match test_glob {
519                        Some(HeaderValue(_, Value::Setting(Setting::Array(test_globs, _, _)))) => {
520                            for setting in test_globs {
521                                match setting {
522                                    Setting::String(test_files, _) => {
523                                        let path_joined = grm_path.parent().unwrap().join(test_files);
524                                        let path_str = &path_joined.to_string_lossy();
525                                        let mut glob_paths = glob(path_str).map_err(|e| e.to_string())?.peekable();
526                                        if glob_paths.peek().is_none() {
527                                            return Err(format!("'test_files' glob '{}' matched no paths", path_str)
528                                                .to_string()
529                                                .into(),
530                                            );
531                                        }
532
533                                        for path in glob_paths {
534                                            let path = path?;
535                                            if let Some(ext) = path.extension()
536                                                && let Some(ext) = ext.to_str()
537                                                    && ext.starts_with("grm") {
538                                                        add_error_line(&mut err_str, "test_files extensions beginning with `grm` are reserved.".into());
539                                                    }
540                                            let input = fs::read_to_string(&path)?;
541                                            let l: LRNonStreamingLexer<LexerTypesT> =
542                                                closure_lexerdef.lexer(&input);
543                                            let errs = rtpb.parse_map(&l, &|_| (), &|_, _| ()).1;
544                                            if !errs.is_empty() {
545                                                add_error_line(&mut err_str, format!("While parsing {}:", path.display()));
546                                                for e in errs {
547                                                    let e_pp = e.pp(&l, &|t| grammar.token_epp(t));
548                                                    let e_lines = e_pp.split("\n");
549                                                    for e in e_lines {
550                                                        add_error_line(&mut err_str, format!("\t{}", e));
551                                                    }
552                                                }
553                                            }
554                                        }
555                                    }
556                                    _ => return Err("Invalid value for setting 'test_files'".into()),
557                                }
558                            }
559                            if let Some(err_str) = err_str {
560                                Err(ErrorString(err_str))?
561                            } else {
562                                Ok(())
563                            }
564
565                        }
566                        Some(_) => Err("Invalid value for setting 'test_files'".into()),
567                        None => Ok(()),
568                    }
569                },
570            ));
571            ctp = lrcfg(ctp);
572            let ct_parser = ctp.build()?;
573            self.rule_ids_map = Some(ct_parser.token_map().to_owned());
574            Some(ct_parser)
575        } else {
576            None
577        };
578
579        let unused_header_values = build_env.header().unused();
580        if !unused_header_values.is_empty() {
581            return Err(
582                format!("Unused header values: {}", unused_header_values.join(", ")).into(),
583            );
584        }
585
586        let code_gen = build_env
587            .code_generator(self.rule_ids_map, env!("VERGEN_BUILD_TIMESTAMP"))
588            .map_err(|e| match e {})?;
589        let (mut missing_from_lexer, missing_from_parser) = {
590            let lexerdef = Box::new(build_env.lexerdef_mut());
591            match code_gen.rule_ids_map() {
592                Some(rim) => {
593                    // Convert from HashMap<String, _> to HashMap<&str, _>
594                    let owned_map = rim
595                        .iter()
596                        .map(|(x, y)| (&**x, *y))
597                        .collect::<HashMap<_, _>>();
598                    let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map);
599                    (
600                        x.map(|a| a.iter().map(|&b| b.to_string()).collect::<HashSet<_>>()),
601                        y.map(|a| {
602                            a.iter()
603                                .map(|(b, span)| (b.to_string(), *span))
604                                .collect::<HashSet<_>>()
605                        }),
606                    )
607                }
608                None => (None, None),
609            }
610        };
611        let lexerdef = build_env.lexerdef();
612        if let Some(mut mfl) = missing_from_lexer.take() {
613            for tok in &lexerdef.expected_missing_tokens {
614                mfl.remove(tok.as_str());
615            }
616            if mfl.is_empty() {
617                missing_from_lexer = None;
618            } else {
619                missing_from_lexer = Some(mfl);
620            }
621        }
622
623        let mut has_unallowed_missing = false;
624        let err_indent = " ".repeat(ERROR.len());
625        if !self.allow_missing_terms_in_lexer
626            && let Some(ref mfl) = missing_from_lexer
627        {
628            if let Some(ct_parser) = &ct_parser {
629                let grm = ct_parser.yacc_grammar();
630                let token_spans = mfl
631                    .iter()
632                    .map(|name| {
633                        ct_parser
634                            .yacc_grammar()
635                            .token_span(*grm.tokens_map().get(name.as_str()).unwrap())
636                            .expect("Given token should have a span")
637                    })
638                    .collect::<Vec<_>>();
639
640                let yacc_diag = SpannedDiagnosticFormatter::new(
641                    ct_parser.grammar_src(),
642                    ct_parser.grammar_path(),
643                );
644
645                eprintln!(
646                    "{ERROR} these tokens are not referenced in the lexer but defined as follows"
647                );
648                eprintln!(
649                    "{err_indent} {}",
650                    yacc_diag.file_location_msg("in the grammar", None)
651                );
652                for span in token_spans {
653                    eprintln!(
654                        "{}",
655                        yacc_diag.underline_span_with_text(
656                            span,
657                            "Missing from lexer".to_string(),
658                            '^'
659                        )
660                    );
661                }
662                eprintln!();
663            } else {
664                eprintln!(
665                    "{ERROR} the following tokens are used in the grammar but are not defined in the lexer:"
666                );
667                for n in mfl {
668                    eprintln!("    {}", n);
669                }
670            }
671            has_unallowed_missing = true;
672        }
673        if !self.allow_missing_tokens_in_parser
674            && self.show_warnings
675            && let Some(ref mfp) = missing_from_parser
676        {
677            let error_prefix = if self.warnings_are_errors {
678                ERROR
679            } else {
680                WARNING
681            };
682            let err_indent = " ".repeat(error_prefix.len());
683            let mut outs = Vec::new();
684            outs.push(format!("{error_prefix} these tokens are not referenced in the grammar but defined as follows"));
685            outs.push(format!(
686                "{err_indent} {}",
687                lex_diag.file_location_msg("in the lexer", None)
688            ));
689            for (_, span) in mfp {
690                let error_contents = lex_diag.underline_span_with_text(
691                    *span,
692                    "Missing from parser".to_string(),
693                    '^',
694                );
695                outs.extend(error_contents.lines().map(|s| s.to_string()));
696            }
697
698            for s in outs {
699                if !self.warnings_are_errors && std::env::var("OUT_DIR").is_ok() {
700                    println!("cargo:warning={}", s)
701                } else {
702                    eprintln!("{}", s);
703                }
704            }
705
706            has_unallowed_missing |= self.warnings_are_errors;
707        }
708        if has_unallowed_missing {
709            fs::remove_file(outp).ok();
710            panic!();
711        }
712        let outs = code_gen
713            .generate(&build_env)
714            .map_err(|e| ErrorString(e.to_string()))?;
715        // If the file we're about to write out already exists with the same contents, then we
716        // don't overwrite it (since that will force a recompile of the file, and relinking of the
717        // binary etc).
718        if let Ok(curs) = read_to_string(outp)
719            && curs == outs
720        {
721            return Ok(CTLexer {
722                missing_from_lexer,
723                missing_from_parser,
724            });
725        }
726        let mut f = File::create(outp)?;
727        f.write_all(outs.as_bytes())?;
728        Ok(CTLexer {
729            missing_from_lexer,
730            missing_from_parser,
731        })
732    }
733
734    /// Given the filename `a/b.l` as input, statically compile the file `src/a/b.l` into a Rust
735    /// module which can then be imported using `lrlex_mod!("a/b.l")`. This is a convenience
736    /// function around [`process_file`](struct.CTLexerBuilder.html#method.process_file) which makes
737    /// it easier to compile `.l` files stored in a project's `src/` directory: please see
738    /// [`process_file`](#method.process_file) for additional constraints and information about the
739    /// generated files.
740    #[deprecated(
741        since = "0.11.0",
742        note = "Please use lexer_in_src_dir() and build() instead"
743    )]
744    #[allow(deprecated)]
745    pub fn process_file_in_src(
746        self,
747        srcp: &str,
748    ) -> Result<(Option<HashSet<String>>, Option<HashSet<String>>), Box<dyn Error>> {
749        let mut inp = current_dir()?;
750        inp.push("src");
751        inp.push(srcp);
752        let mut outp = PathBuf::new();
753        outp.push(var("OUT_DIR").unwrap());
754        outp.push(Path::new(srcp).parent().unwrap().to_str().unwrap());
755        create_dir_all(&outp)?;
756        let mut leaf = Path::new(srcp)
757            .file_name()
758            .unwrap()
759            .to_str()
760            .unwrap()
761            .to_owned();
762        write!(leaf, ".{}", RUST_FILE_EXT).ok();
763        outp.push(leaf);
764        self.process_file(inp, outp)
765    }
766
767    /// Statically compile the `.l` file `inp` into Rust, placing the output into the file `outp`.
768    /// The latter defines a module as follows:
769    ///
770    /// ```text
771    ///    mod modname {
772    ///      pub fn lexerdef() -> LexerDef<LexerTypesT::StorageT> { ... }
773    ///
774    ///      ...
775    ///    }
776    /// ```
777    ///
778    /// where:
779    ///  * `modname` is either:
780    ///    * the module name specified [`mod_name`](#method.mod_name)
781    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.l` the
782    ///      module name is `c_l` (i.e. the file's leaf name, minus its extension, with a prefix of
783    ///      `_l`).
784    #[deprecated(
785        since = "0.11.0",
786        note = "Please use lexer_in_src_dir() and build() instead"
787    )]
788    pub fn process_file<P, Q>(
789        mut self,
790        inp: P,
791        outp: Q,
792    ) -> Result<(Option<HashSet<String>>, Option<HashSet<String>>), Box<dyn Error>>
793    where
794        P: AsRef<Path>,
795        Q: AsRef<Path>,
796    {
797        self.lexer_path = Some(inp.as_ref().to_owned());
798        self.output_path = Some(outp.as_ref().to_owned());
799        let cl = self.build()?;
800        Ok((
801            cl.missing_from_lexer().map(|x| x.to_owned()),
802            cl.missing_from_parser()
803                .map(|x| x.iter().map(|(n, _)| n.to_owned()).collect::<HashSet<_>>()),
804        ))
805    }
806
807    /// If passed false, tokens used in the grammar but not defined in the lexer will cause a
808    /// panic at lexer generation time. Defaults to false.
809    pub fn allow_missing_terms_in_lexer(mut self, allow: bool) -> Self {
810        self.allow_missing_terms_in_lexer = allow;
811        self
812    }
813
814    /// If passed false, tokens defined in the lexer but not used in the grammar will cause a
815    /// warning at lexer generation time. Defaults to false (since lexers sometimes define tokens such
816    /// as reserved words, which are intentionally not in the grammar).
817    pub fn allow_missing_tokens_in_parser(mut self, allow: bool) -> Self {
818        self.allow_missing_tokens_in_parser = allow;
819        self
820    }
821
822    /// If set to true, [CTLexerBuilder::build] will return an error if the given lexer contains
823    /// any warnings. Defaults to `true`.
824    pub fn warnings_are_errors(mut self, flag: bool) -> Self {
825        self.warnings_are_errors = flag;
826        self
827    }
828
829    /// If set to true, [CTParserBuilder::build] will print warnings to stderr, or via cargo when
830    /// running under cargo. Defaults to `true`.
831    pub fn show_warnings(mut self, flag: bool) -> Self {
832        self.show_warnings = flag;
833        self
834    }
835
836    /// Enables `// comment` style parsing according to `flag``.
837    /// When enabled comments can appear at the beginning of a line,
838    /// and regular expressions with the `/` character should be escaped via `\/`.
839    ///
840    /// The default value is `false`.
841    ///
842    /// Setting this flag will override the same flag within a `%grmtools` section.
843    pub fn allow_wholeline_comments(mut self, flag: bool) -> Self {
844        let key = "lrlex.allow_wholeline_comments".to_string();
845        self.header.insert(
846            key,
847            HeaderValue(
848                Location::Other("CTLexerBuilder".to_string()),
849                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
850            ),
851        );
852        self
853    }
854
855    /// Sets the `regex::RegexBuilder` option of the same name.
856    /// The default value is `true`.
857    ///
858    /// Setting this flag will override the same flag within a `%grmtools` section.
859    pub fn dot_matches_new_line(mut self, flag: bool) -> Self {
860        let key = "regex.dot_matches_new_line".to_string();
861        self.header.insert(
862            key,
863            HeaderValue(
864                Location::Other("CTLexerBuilder".to_string()),
865                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
866            ),
867        );
868        self
869    }
870
871    /// Sets the `regex::RegexBuilder` option of the same name.
872    /// The default value is `true`.
873    ///
874    /// Setting this flag will override the same flag within a `%grmtools` section.
875    pub fn multi_line(mut self, flag: bool) -> Self {
876        let key = "regex.multi_line".to_string();
877        self.header.insert(
878            key,
879            HeaderValue(
880                Location::Other("CTLexerBuilder".to_string()),
881                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
882            ),
883        );
884        self
885    }
886
887    /// Enables posix lex compatible escape sequences according to `flag`.
888    /// The default value is `false`.
889    ///
890    /// Setting this flag will override the same flag within a `%grmtools` section.
891    pub fn posix_escapes(mut self, flag: bool) -> Self {
892        let key = "lrlex.posix_escapes".to_string();
893        self.header.insert(
894            key,
895            HeaderValue(
896                Location::Other("CTLexerBuilder".to_string()),
897                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
898            ),
899        );
900        self
901    }
902
903    /// Sets the `regex::RegexBuilder` option of the same name.
904    /// The default value is `true`.
905    ///
906    /// Setting this flag will override the same flag within a `%grmtools` section.
907    pub fn octal(mut self, flag: bool) -> Self {
908        let key = "regex.octal".to_string();
909        self.header.insert(
910            key,
911            HeaderValue(
912                Location::Other("CTLexerBuilder".to_string()),
913                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
914            ),
915        );
916        self
917    }
918
919    /// Sets the `regex::RegexBuilder` option of the same name.
920    /// Default value is specified by regex.
921    ///
922    /// Setting this flag will override the same flag within a `%grmtools` section.
923    pub fn swap_greed(mut self, flag: bool) -> Self {
924        let key = "regex.swap_greed".to_string();
925        self.header.insert(
926            key,
927            HeaderValue(
928                Location::Other("CTLexerBuilder".to_string()),
929                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
930            ),
931        );
932        self
933    }
934
935    /// Sets the `regex::RegexBuilder` option of the same name.
936    /// Default value is specified by regex.
937    ///
938    /// Setting this flag will override the same flag within a `%grmtools` section.
939    pub fn ignore_whitespace(mut self, flag: bool) -> Self {
940        let key = "regex.ignore_whitespace".to_string();
941        self.header.insert(
942            key,
943            HeaderValue(
944                Location::Other("CTLexerBuilder".to_string()),
945                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
946            ),
947        );
948        self
949    }
950
951    /// Sets the `regex::RegexBuilder` option of the same name.
952    /// Default value is specified by regex.
953    ///
954    /// Setting this flag will override the same flag within a `%grmtools` section.
955    pub fn unicode(mut self, flag: bool) -> Self {
956        let key = "regex.unicode".to_string();
957        self.header.insert(
958            key,
959            HeaderValue(
960                Location::Other("CTLexerBuilder".to_string()),
961                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
962            ),
963        );
964        self
965    }
966
967    /// Sets the `regex::RegexBuilder` option of the same name.
968    /// Default value is specified by regex.
969    ///
970    /// Setting this flag will override the same flag within a `%grmtools` section.
971    pub fn case_insensitive(mut self, flag: bool) -> Self {
972        let key = "regex.case_insensitive".to_string();
973        self.header.insert(
974            key,
975            HeaderValue(
976                Location::Other("CTLexerBuilder".to_string()),
977                Value::Flag(flag, Location::Other("CTLexerBuilder".to_string())),
978            ),
979        );
980        self
981    }
982
983    /// Sets the `regex::RegexBuilder` option of the same name.
984    /// Default value is specified by regex.
985    ///
986    /// Setting this flag will override the same flag within a `%grmtools` section.
987    pub fn size_limit(mut self, sz: usize) -> Self {
988        let key = "regex.size_limit".to_string();
989        self.header.insert(
990            key,
991            HeaderValue(
992                Location::Other("CTLexerBuilder".to_string()),
993                Value::Setting(Setting::Num(
994                    sz as u64,
995                    Location::Other("CTLexerBuilder".to_string()),
996                )),
997            ),
998        );
999        self
1000    }
1001
1002    /// Sets the `regex::RegexBuilder` option of the same name.
1003    /// Default value is specified by regex.
1004    ///
1005    /// Setting this flag will override the same flag within a `%grmtools` section.
1006    pub fn dfa_size_limit(mut self, sz: usize) -> Self {
1007        let key = "regex.dfa_size_limit".to_string();
1008        self.header.insert(
1009            key,
1010            HeaderValue(
1011                Location::Other("CTLexerBuilder".to_string()),
1012                Value::Setting(Setting::Num(
1013                    sz as u64,
1014                    Location::Other("CTLexerBuilder".to_string()),
1015                )),
1016            ),
1017        );
1018        self
1019    }
1020
1021    /// Sets the `regex::RegexBuilder` option of the same name.
1022    /// Default value is specified by regex.
1023    ///
1024    /// Setting this flag will override the same flag within a `%grmtools` section.
1025    pub fn nest_limit(mut self, lim: u32) -> Self {
1026        let key = "regex.nest_limit".to_string();
1027        self.header.insert(
1028            key,
1029            HeaderValue(
1030                Location::Other("CTLexerBuilder".to_string()),
1031                Value::Setting(Setting::Num(
1032                    lim as u64,
1033                    Location::Other("CTLexerBuilder".to_string()),
1034                )),
1035            ),
1036        );
1037        self
1038    }
1039
1040    #[cfg(test)]
1041    pub fn inspect_lexerkind(
1042        mut self,
1043        cb: Box<dyn Fn(&LexerKind) -> Result<(), Box<dyn Error>>>,
1044    ) -> Self {
1045        self.inspect_lexerkind_cb = Some(cb);
1046        self
1047    }
1048}
1049
1050/// An interface to the result of [CTLexerBuilder::build()].
1051pub struct CTLexer {
1052    missing_from_lexer: Option<HashSet<String>>,
1053    missing_from_parser: Option<HashSet<(String, Span)>>,
1054}
1055
1056impl CTLexer {
1057    fn missing_from_lexer(&self) -> Option<&HashSet<String>> {
1058        self.missing_from_lexer.as_ref()
1059    }
1060
1061    fn missing_from_parser(&self) -> Option<&HashSet<(String, Span)>> {
1062        self.missing_from_parser.as_ref()
1063    }
1064}
1065
1066/// Exports all token IDs used by a parser as a separate Rust module.
1067///
1068/// This builder will create a Rust module named `mod_name`
1069/// that can be imported with [`lrlex_mod!(mod_name)`](crate::lrlex_mod).
1070/// The module will contain one `const` `StorageT` per token in `token_map`,
1071/// with the token prefixed by `T_`. In addition, it will contain
1072/// an array of all token IDs `TOK_IDS`.
1073///
1074/// For example, if `StorageT` is `u8`, `mod_name` is `x`, and `token_map` is
1075/// `HashMap{"ID": 0, "INT": 1}` the generated module will look roughly as follows:
1076///
1077/// ```rust,ignore
1078/// mod x {
1079///   pub const T_ID: u8 = 0;
1080///   pub const T_INT: u8 = 1;
1081///   pub const TOK_IDS: &[u8] = &[T_ID, T_INT];
1082/// }
1083/// ```
1084///
1085/// See the [custom lexer example] for more usage details.
1086///
1087/// [custom lexer example]: https://github.com/softdevteam/grmtools/tree/master/lrlex/examples/calc_manual_lex
1088#[derive(Debug, Clone)]
1089pub struct CTTokenMapBuilder<StorageT: Display + ToTokens> {
1090    mod_name: String,
1091    token_map: Vec<(String, TokenStream)>,
1092    rename_map: Option<HashMap<String, String>>,
1093    allow_dead_code: bool,
1094    _marker: PhantomData<StorageT>,
1095}
1096
1097impl<StorageT: Display + ToTokens> CTTokenMapBuilder<StorageT> {
1098    /// Create a new token map builder.
1099    ///
1100    /// See the [builder documentation] for more info.
1101    ///
1102    /// [builder documentation]: CTTokenMapBuilder
1103    pub fn new(
1104        mod_name: impl Into<String>,
1105        token_map: impl Borrow<HashMap<String, StorageT>>,
1106    ) -> Self {
1107        Self {
1108            mod_name: mod_name.into(),
1109            token_map: token_map
1110                .borrow()
1111                .iter()
1112                .map(|(tok_name, tok_value)| (tok_name.clone(), tok_value.to_token_stream()))
1113                .collect(),
1114            rename_map: None,
1115            allow_dead_code: false,
1116            _marker: PhantomData,
1117        }
1118    }
1119
1120    /// Set a token rename map.
1121    ///
1122    /// Rename map is used to specify identifier names for tokens whose names
1123    /// are not valid Rust identifiers. For example, if `token_map`
1124    /// is `HashMap{"+": 0, "ID": 1}` and `rename_map` is `HashMap{"+": "PLUS"}`
1125    /// then the generated module will look roughly as follows:
1126    ///
1127    /// ```rust,ignore
1128    /// mod x {
1129    ///   pub const T_PLUS: u8 = 0;
1130    ///   pub const T_ID: u8 = 1;
1131    /// }
1132    /// ```
1133    pub fn rename_map<M, I, K, V>(mut self, rename_map: Option<M>) -> Self
1134    where
1135        M: IntoIterator<Item = I>,
1136        I: Borrow<(K, V)>,
1137        K: AsRef<str>,
1138        V: AsRef<str>,
1139    {
1140        self.rename_map = rename_map.map(|rename_map| {
1141            rename_map
1142                .into_iter()
1143                .map(|it| {
1144                    let (k, v) = it.borrow();
1145                    let k = k.as_ref().into();
1146                    let v = v.as_ref().into();
1147                    (k, v)
1148                })
1149                .collect()
1150        });
1151        self
1152    }
1153
1154    /// Control whether the builder will add `#[allow(dead_code)]`
1155    /// to the generated module.
1156    ///
1157    /// By default, all tokens are `#[deny(dead_code)]`, meaning that you'll
1158    /// get a warning if your custom lexer doesn't use any of them.
1159    /// This function can be used to disable this behavior.
1160    pub fn allow_dead_code(mut self, allow_dead_code: bool) -> Self {
1161        self.allow_dead_code = allow_dead_code;
1162        self
1163    }
1164
1165    /// Build the token map module.
1166    pub fn build(&self) -> Result<(), Box<dyn Error>> {
1167        // Record the time that this version of lrlex was built. If the source code changes and rustc
1168        // forces a recompile, this will change this value, causing anything which depends on this
1169        // build of lrlex to be recompiled too.
1170        let mut outs = String::new();
1171        let timestamp = env!("VERGEN_BUILD_TIMESTAMP");
1172        let mod_ident = format_ident!("{}", self.mod_name);
1173        write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok();
1174        let storaget = str::parse::<TokenStream>(type_name::<StorageT>()).unwrap();
1175        // Sort the tokens so that they're always in the same order.
1176        // This will prevent unneeded rebuilds.
1177        let mut token_map_sorted = self.token_map.clone();
1178        token_map_sorted.sort_by(|(l, _), (r, _)| l.cmp(r));
1179        let (token_array, tokens) = token_map_sorted
1180            .iter()
1181            .map(|(k, id)| {
1182                let name = match &self.rename_map {
1183                    Some(rmap) => rmap.get(k).unwrap_or(k),
1184                    _ => k,
1185                };
1186                let tok_ident: Ident = syn::parse_str(&format!("T_{}", name.to_ascii_uppercase()))
1187                    .map_err(|e| {
1188                        format!(
1189                            "token name {:?} is not a valid Rust identifier: {}; \
1190                            consider renaming it via `CTTokenMapBuilder::rename_map`.",
1191                            name, e
1192                        )
1193                    })?;
1194                Ok((
1195                    // Note: the array of all tokens can't use `tok_ident` because
1196                    // it will confuse the dead code checker. For this reason,
1197                    // we use `id` here.
1198                    quote! {
1199                        #id,
1200                    },
1201                    quote! {
1202                        pub const #tok_ident: #storaget = #id;
1203                    },
1204                ))
1205            })
1206            .collect::<Result<(TokenStream, TokenStream), Box<dyn Error>>>()?;
1207        let unused_annotation = if self.allow_dead_code {
1208            quote! {#[allow(dead_code)]}
1209        } else {
1210            quote! {}
1211        };
1212        // Since the formatter doesn't preserve comments and we don't want to lose build time,
1213        // just format the module contents.
1214        let unformatted = quote! {
1215            #unused_annotation
1216            mod #mod_ident {
1217                #tokens
1218                #[allow(dead_code)]
1219                pub const TOK_IDS: &[#storaget] = &[#token_array];
1220            }
1221        }
1222        .to_string();
1223        let out_mod = syn::parse_str(&unformatted)
1224            .map(|syntax_tree| prettyplease::unparse(&syntax_tree))
1225            .unwrap_or(unformatted);
1226        outs.push_str(&out_mod);
1227        let mut outp = PathBuf::from(var("OUT_DIR")?);
1228        outp.push(&self.mod_name);
1229        outp.set_extension("rs");
1230
1231        // If the file we're about to write out already exists with the same contents, then we
1232        // don't overwrite it (since that will force a recompile of the file, and relinking of the
1233        // binary etc).
1234        if let Ok(curs) = read_to_string(&outp)
1235            && curs == outs
1236        {
1237            return Ok(());
1238        }
1239
1240        let mut f = File::create(outp)?;
1241        f.write_all(outs.as_bytes())?;
1242        Ok(())
1243    }
1244}
1245
1246/// Create a Rust module named `mod_name` that can be imported with
1247/// [`lrlex_mod!(mod_name)`](crate::lrlex_mod).
1248///
1249/// This function is deprecated in favour of [`CTTokenMapBuilder`].
1250#[deprecated(since = "0.14.0", note = "use `lrlex::CTTokenMapBuilder` instead")]
1251pub fn ct_token_map<StorageT: Display + ToTokens>(
1252    mod_name: &str,
1253    token_map: impl Borrow<HashMap<String, StorageT>>,
1254    rename_map: Option<&HashMap<&str, &str>>,
1255) -> Result<(), Box<dyn Error>> {
1256    CTTokenMapBuilder::new(mod_name, token_map)
1257        .rename_map(rename_map)
1258        .allow_dead_code(true)
1259        .build()
1260}
1261
1262/// Indents a multi-line string and trims any trailing newline.
1263/// This currently assumes that indentation on blank lines does not matter.
1264///
1265/// The algorithm used by this function is:
1266/// 1. Prefix `s` with the indentation, indenting the first line.
1267/// 2. Trim any trailing newlines.
1268/// 3. Replace all newlines with `\n{indent}`` to indent all lines after the first.
1269///
1270/// It is plausible that we should a step 4, but currently do not:
1271/// 4. Replace all `\n{indent}\n` with `\n\n`
1272fn indent(indent: &str, s: &str) -> String {
1273    format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent))
1274}
1275
1276// It isn't clear to me why this test isn't working on wasm32,
1277// as the `workspace_runner` should allow access to `OUT_DIR`
1278// perhaps it is related to absolute paths
1279#[cfg(all(not(target_arch = "wasm32"), test))]
1280mod test {
1281    use std::fs::File;
1282    use std::io::Write;
1283
1284    use super::{CTLexerBuilder, LexerKind};
1285    #[test]
1286    fn test_grmtools_section_lexerkind() {
1287        let lexerkinds = [
1288            "LRNonStreamingLexer",
1289            "lrnonstreaminglexer",
1290            "LexerKind::lrnonstreaminglexer",
1291            "lexerkind::LRNonStreamingLexer",
1292        ];
1293        for (i, kind) in lexerkinds.iter().enumerate() {
1294            let lex_src = format!(
1295                "
1296%grmtools{{lexerkind: {}}}
1297%%
1298. ;
1299",
1300                kind
1301            );
1302            let lex_path = format!(
1303                "{}/test_grmtools_section_lexerkind_{}.l",
1304                env!("OUT_DIR"),
1305                i
1306            );
1307            let mut l_file = File::create(lex_path.clone()).unwrap();
1308            l_file.write_all(lex_src.as_bytes()).unwrap();
1309            CTLexerBuilder::new()
1310                .output_path(format!("{}.rs", lex_path.clone()))
1311                .lexer_path(lex_path.clone())
1312                .inspect_lexerkind(Box::new(move |lexerkind| {
1313                    assert!(matches!(lexerkind, &LexerKind::LRNonStreamingLexer));
1314                    Ok(())
1315                }))
1316                .build()
1317                .unwrap();
1318        }
1319    }
1320
1321    #[test]
1322    /// Tests a yacc .y filename containing a dash character leading to an invalid rust identifier
1323    /// when that dash is subsequently used as the default `CTParserBuilder::mod_name`.
1324    fn test_invalid_identifier_in_derived_mod_name() {
1325        let mut lex_path = std::path::PathBuf::from(env!("OUT_DIR"));
1326        lex_path.push("contains-a-dash.l");
1327        let mut f = File::create(&lex_path).unwrap();
1328        let _ = f.write_all(
1329            r#"
1330%%
1331A  "A"
1332"#
1333            .as_bytes(),
1334        );
1335        match CTLexerBuilder::new()
1336            .output_path(format!("{}.rs", lex_path.display()))
1337            .lexer_path(lex_path.clone())
1338            .build()
1339        {
1340            Ok(_) => panic!("Expected error"),
1341            Err(e) => {
1342                let err_string = e.to_string();
1343                assert_eq!(
1344                    err_string,
1345                    "mod_name 'contains-a-dash_l' is not a valid rust identifier due to 'unexpected token'"
1346                );
1347            }
1348        }
1349    }
1350}
lrlex/ctbuilder.rs

lrlex/
ctbuilder.rs
