lrlex/
ctbuilder.rs

1//! Build grammars at run-time.
2
3use cfgrammar::{
4    header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Value},
5    markmap::MergeBehavior,
6    span::{Location, Span},
7};
8use glob::glob;
9use lrpar::{
10    CTParserBuilder, LexerTypes,
11    diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter},
12};
13use num_traits::{AsPrimitive, PrimInt, Unsigned};
14use proc_macro2::{Ident, TokenStream};
15use quote::{ToTokens, format_ident, quote};
16use std::{
17    any::type_name,
18    borrow::Borrow,
19    collections::{HashMap, HashSet},
20    env::{current_dir, var},
21    error::Error,
22    fmt::{self, Debug, Display, Write as _},
23    fs::{self, File, create_dir_all, read_to_string},
24    hash::Hash,
25    io::Write,
26    marker::PhantomData,
27    path::{Path, PathBuf},
28    sync::{LazyLock, Mutex},
29};
30use wincode::SchemaWrite;
31
32use crate::{
33    DefaultLexerTypes, LRNonStreamingLexer, LexerDef,
34    codegen::{LexerBuildEnvArgs, LexerSrcEnv, LexerSrcEnvError},
35};
36
37const RUST_FILE_EXT: &str = "rs";
38
39const ERROR: &str = "[Error]";
40const WARNING: &str = "[Warning]";
41
42static GENERATED_PATHS: LazyLock<Mutex<HashSet<PathBuf>>> =
43    LazyLock::new(|| Mutex::new(HashSet::new()));
44
45#[non_exhaustive]
46pub enum LexerKind {
47    LRNonStreamingLexer,
48}
49
50impl<T: Clone> TryFrom<&Value<T>> for LexerKind {
51    type Error = cfgrammar::header::HeaderError<T>;
52    fn try_from(it: &Value<T>) -> Result<LexerKind, Self::Error> {
53        match it {
54            Value::Namespaced(rs, loc) => match rs.as_str() {
55                "LexerKind::LRNonStreamingLexer" | "LRNonStreamingLexer" => {
56                    Ok(LexerKind::LRNonStreamingLexer)
57                }
58                _ => Err(HeaderError {
59                    kind: HeaderErrorKind::ConversionError("LexerKind", "Expected `LexerKind`"),
60                    locations: vec![loc.clone()],
61                }),
62            },
63            val => Err(HeaderError {
64                kind: HeaderErrorKind::ConversionError("LexerKind", "Expected `LexerKind`"),
65                locations: vec![val.primary_location().clone()],
66            }),
67        }
68    }
69}
70
71/// Specify the visibility of the module generated by [CTLexerBuilder].
72#[derive(Clone, PartialEq, Eq, Debug)]
73#[non_exhaustive]
74pub enum Visibility {
75    /// Module-level visibility only.
76    Private,
77    /// `pub`
78    Public,
79    /// `pub(super)`
80    PublicSuper,
81    /// `pub(self)`
82    PublicSelf,
83    /// `pub(crate)`
84    PublicCrate,
85    /// `pub(in {arg})`
86    PublicIn(String),
87}
88
89impl ToTokens for Visibility {
90    fn to_tokens(&self, tokens: &mut TokenStream) {
91        tokens.extend(match self {
92            Visibility::Private => quote!(),
93            Visibility::Public => quote! {pub},
94            Visibility::PublicSuper => quote! {pub(super)},
95            Visibility::PublicSelf => quote! {pub(self)},
96            Visibility::PublicCrate => quote! {pub(crate)},
97            Visibility::PublicIn(data) => {
98                let other = str::parse::<TokenStream>(data).unwrap();
99                quote! {pub(in #other)}
100            }
101        })
102    }
103}
104
105/// Specifies the [Rust Edition] that will be emitted during code generation.
106///
107/// [Rust Edition]: https://doc.rust-lang.org/edition-guide/rust-2021/index.html
108#[derive(Clone, Copy, PartialEq, Eq, Debug)]
109#[non_exhaustive]
110pub enum RustEdition {
111    Rust2015,
112    Rust2018,
113    Rust2021,
114}
115
116/// A string which uses `Display` for it's `Debug` impl.
117struct ErrorString(String);
118impl fmt::Display for ErrorString {
119    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
120        let ErrorString(s) = self;
121        write!(f, "{}", s)
122    }
123}
124impl fmt::Debug for ErrorString {
125    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
126        let ErrorString(s) = self;
127        write!(f, "{}", s)
128    }
129}
130impl Error for ErrorString {}
131
132/// A `CTLexerBuilder` allows one to specify the criteria for building a statically generated
133/// lexer.
134pub struct CTLexerBuilder<'a, LexerTypesT: LexerTypes = DefaultLexerTypes<u32>>
135where
136    LexerTypesT::StorageT: Debug + Eq + Hash + ToTokens,
137    usize: num_traits::AsPrimitive<LexerTypesT::StorageT>,
138{
139    lrpar_config:
140        Option<Box<dyn Fn(CTParserBuilder<LexerTypesT>) -> CTParserBuilder<LexerTypesT> + 'a>>,
141    lexer_path: Option<PathBuf>,
142    output_path: Option<PathBuf>,
143    lexerkind: Option<LexerKind>,
144    mod_name: Option<&'a str>,
145    visibility: Visibility,
146    rust_edition: RustEdition,
147    rule_ids_map: Option<HashMap<String, LexerTypesT::StorageT>>,
148    allow_missing_terms_in_lexer: bool,
149    allow_missing_tokens_in_parser: bool,
150    warnings_are_errors: bool,
151    show_warnings: bool,
152    header: Header<Location>,
153    #[cfg(test)]
154    inspect_lexerkind_cb: Option<Box<dyn Fn(&LexerKind) -> Result<(), Box<dyn Error>>>>,
155}
156
157impl CTLexerBuilder<'_, DefaultLexerTypes<u32>> {
158    /// Create a new [CTLexerBuilder].
159    pub fn new() -> Self {
160        CTLexerBuilder::<DefaultLexerTypes<u32>>::new_with_lexemet()
161    }
162}
163type FixIntConfig = wincode::config::Configuration;
164
165type VarIntConfig = wincode::config::Configuration<
166    true,
167    4194304,
168    wincode::len::BincodeLen,
169    wincode::int_encoding::LittleEndian,
170    wincode::int_encoding::VarInt,
171>;
172impl<'a, LexerTypesT: LexerTypes<LexErrorT = crate::LRLexError> + 'static>
173    CTLexerBuilder<'a, LexerTypesT>
174where
175    LexerTypesT::StorageT: 'static
176        + Debug
177        + Eq
178        + Hash
179        + PrimInt
180        + SchemaWrite<FixIntConfig, Src = LexerTypesT::StorageT>
181        + SchemaWrite<VarIntConfig, Src = LexerTypesT::StorageT>
182        + TryFrom<usize>
183        + Unsigned
184        + ToTokens,
185    usize: AsPrimitive<LexerTypesT::StorageT>,
186{
187    /// Create a new [CTLexerBuilder].
188    ///
189    /// `LexerTypesT::StorageT` must be an unsigned integer type (e.g. `u8`, `u16`) which is big enough
190    /// to index all the tokens, rules, and productions in the lexer and less than or equal in size
191    /// to `usize` (e.g. on a 64-bit machine `u128` would be too big). If you are lexing large
192    /// files, the additional storage requirements of larger integer types can be noticeable, and
193    /// in such cases it can be worth specifying a smaller type. `StorageT` defaults to `u32` if
194    /// unspecified.
195    ///
196    /// # Examples
197    ///
198    /// ```text
199    /// CTLexerBuilder::<DefaultLexerTypes<u8>>::new_with_lexemet()
200    ///     .lexer_in_src_dir("grm.l", None)?
201    ///     .build()?;
202    /// ```
203    pub fn new_with_lexemet() -> Self {
204        let mut header = Header::new();
205        header.set_default_merge_behavior(MergeBehavior::Ours);
206        CTLexerBuilder {
207            lrpar_config: None,
208            lexer_path: None,
209            output_path: None,
210            lexerkind: None,
211            mod_name: None,
212            visibility: Visibility::Private,
213            rust_edition: RustEdition::Rust2021,
214            rule_ids_map: None,
215            allow_missing_terms_in_lexer: false,
216            allow_missing_tokens_in_parser: false,
217            warnings_are_errors: false,
218            show_warnings: true,
219            header,
220            #[cfg(test)]
221            inspect_lexerkind_cb: None,
222        }
223    }
224
225    /// An optional convenience function to make it easier to create an (lrlex) lexer and (lrpar)
226    /// parser in one shot. The closure passed to this function will be called during
227    /// [CTLexerBuilder::build]: it will be passed an lrpar `CTParserBuilder` instance upon which
228    /// it can set whatever lrpar options are desired. [`CTLexerBuilder`] will then create both the
229    /// compiler and lexer and link them together as required.
230    ///
231    /// # Examples
232    ///
233    /// ```text
234    /// CTLexerBuilder:::new()
235    ///     .lrpar_config(|ctp| {
236    ///         ctp.yacckind(YaccKind::Grmtools)
237    ///             .grammar_in_src_dir("calc.y")
238    ///             .unwrap()
239    ///     })
240    ///     .lexer_in_src_dir("calc.l")?
241    ///     .build()?;
242    /// ```
243    pub fn lrpar_config<F>(mut self, config_func: F) -> Self
244    where
245        F: Fn(CTParserBuilder<LexerTypesT>) -> CTParserBuilder<LexerTypesT> + 'a,
246    {
247        self.lrpar_config = Some(Box::new(config_func));
248        self
249    }
250
251    /// Set the input lexer path to a file relative to this project's `src` directory. This will
252    /// also set the output path (i.e. you do not need to call [CTLexerBuilder::output_path]).
253    ///
254    /// For example if `a/b.l` is passed as `inp` then [CTLexerBuilder::build] will:
255    ///   * use `src/a/b.l` as the input file.
256    ///   * write output to a file which can then be imported by calling `lrlex_mod!("a/b.l")`.
257    ///   * create a module in that output file named `b_l`.
258    ///
259    /// You can override the output path and/or module name by calling
260    /// [CTLexerBuilder::output_path] and/or [CTLexerBuilder::mod_name], respectively, after
261    /// calling this function.
262    ///
263    /// This is a convenience function that makes it easier to compile lexer files stored in a
264    /// project's `src/` directory: please see [CTLexerBuilder::build] for additional constraints
265    /// and information about the generated files. Note also that each `.l` file can only be
266    /// processed once using this function: if you want to generate multiple lexers from a single
267    /// `.l` file, you will need to use [CTLexerBuilder::output_path].
268    pub fn lexer_in_src_dir<P>(mut self, srcp: P) -> Result<Self, Box<dyn Error>>
269    where
270        P: AsRef<Path>,
271    {
272        if !srcp.as_ref().is_relative() {
273            return Err(format!(
274                "Lexer path '{}' must be a relative path.",
275                srcp.as_ref().to_str().unwrap_or("<invalid UTF-8>")
276            )
277            .into());
278        }
279
280        let mut lexp = current_dir()?;
281        lexp.push("src");
282        lexp.push(srcp.as_ref());
283        self.lexer_path = Some(lexp);
284
285        let mut outp = PathBuf::new();
286        outp.push(var("OUT_DIR").unwrap());
287        outp.push(srcp.as_ref().parent().unwrap().to_str().unwrap());
288        create_dir_all(&outp)?;
289        let mut leaf = srcp
290            .as_ref()
291            .file_name()
292            .unwrap()
293            .to_str()
294            .unwrap()
295            .to_owned();
296        write!(leaf, ".{}", RUST_FILE_EXT).ok();
297        outp.push(leaf);
298        Ok(self.output_path(outp))
299    }
300
301    /// Set the input lexer path to `inp`. If specified, you must also call
302    /// [CTLexerBuilder::output_path]. In general it is easier to use
303    /// [CTLexerBuilder::lexer_in_src_dir].
304    pub fn lexer_path<P>(mut self, inp: P) -> Self
305    where
306        P: AsRef<Path>,
307    {
308        self.lexer_path = Some(inp.as_ref().to_owned());
309        self
310    }
311
312    /// Set the output lexer path to `outp`. Note that there are no requirements on `outp`: the
313    /// file can exist anywhere you can create a valid [Path] to. However, if you wish to use
314    /// [crate::lrlex_mod!] you will need to make sure that `outp` is in
315    /// [std::env::var]`("OUT_DIR")` or one of its subdirectories.
316    pub fn output_path<P>(mut self, outp: P) -> Self
317    where
318        P: AsRef<Path>,
319    {
320        self.output_path = Some(outp.as_ref().to_owned());
321        self
322    }
323
324    /// Set the type of lexer to be generated to `lexerkind`.
325    pub fn lexerkind(mut self, lexerkind: LexerKind) -> Self {
326        self.lexerkind = Some(lexerkind);
327        self
328    }
329
330    /// Set the generated module name to `mod_name`. If no module name is specified,
331    /// [`process_file`](#method.process_file) will attempt to create a sensible default based on
332    /// the input filename.
333    pub fn mod_name(mut self, mod_name: &'a str) -> Self {
334        self.mod_name = Some(mod_name);
335        self
336    }
337
338    /// Set the visibility of the generated module to `vis`. Defaults to `Visibility::Private`.
339    pub fn visibility(mut self, vis: Visibility) -> Self {
340        self.visibility = vis;
341        self
342    }
343
344    /// Sets the rust edition to be used for generated code. Defaults to the latest edition of
345    /// rust supported by grmtools.
346    pub fn rust_edition(mut self, edition: RustEdition) -> Self {
347        self.rust_edition = edition;
348        self
349    }
350
351    /// Set this lexer builder's map of rule IDs to `rule_ids_map`. By default, lexing rules have
352    /// arbitrary, but distinct, IDs. Setting the map of rule IDs (from rule names to `StorageT`)
353    /// allows users to synchronise a lexer and parser and to check that all rules are used by both
354    /// parts).
355    pub fn rule_ids_map<T: std::borrow::Borrow<HashMap<String, LexerTypesT::StorageT>> + Clone>(
356        mut self,
357        rule_ids_map: T,
358    ) -> Self {
359        self.rule_ids_map = Some(rule_ids_map.borrow().to_owned());
360        self
361    }
362
363    /// Statically compile the `.l` file specified by [CTLexerBuilder::lexer_path()] into Rust,
364    /// placing the output into the file specified by [CTLexerBuilder::output_path()].
365    ///
366    /// The generated module follows the form:
367    ///
368    /// ```text
369    ///    mod modname {
370    ///      pub fn lexerdef() -> LexerDef<LexerTypesT> { ... }
371    ///
372    ///      ...
373    ///    }
374    /// ```
375    ///
376    /// where:
377    ///  * `modname` is either:
378    ///    * the module name specified by [CTLexerBuilder::mod_name()]
379    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.l` the
380    ///      module name is `c_l` (i.e. the file's leaf name, minus its extension, with a prefix of
381    ///      `_l`).
382    pub fn build(mut self) -> Result<CTLexer, Box<dyn Error>> {
383        let lexerp = self
384            .lexer_path
385            .as_ref()
386            .expect("lexer_path must be specified before processing.");
387        let outp = self
388            .output_path
389            .as_ref()
390            .expect("output_path must be specified before processing.");
391
392        {
393            let mut lk = GENERATED_PATHS.lock().unwrap();
394            if lk.contains(outp.as_path()) {
395                return Err(format!("Generating two lexers to the same path ('{}') is not allowed: use CTLexerBuilder::output_path (and, optionally, CTLexerBuilder::mod_name) to differentiate them.", outp.to_str().unwrap()).into());
396            }
397            lk.insert(outp.clone());
398        }
399        let lex_src = read_to_string(lexerp)
400            .map_err(|e| format!("When reading '{}': {e}", lexerp.display()))?;
401        let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, lexerp);
402        let args = LexerBuildEnvArgs::new()
403            .mod_name(self.mod_name.map(|s| s.to_string()))
404            .lexerkind(self.lexerkind)
405            .visibility(self.visibility);
406        let mut build_env =
407            LexerSrcEnv::<LexerTypesT>::new_with_header(&lex_src, Some(lexerp), self.header)
408                .build_env(args)
409                .map_err(|e| match e {
410                    LexerSrcEnvError::GrmtoolsSectionParseError(es) => {
411                        let mut out = String::new();
412                        out.push_str(&format!(
413                            "\n{ERROR}{}\n",
414                            lex_diag.file_location_msg(" parsing the `%grmtools` section", None)
415                        ));
416                        for e in es {
417                            out.push_str(&indent("     ", &lex_diag.format_error(e).to_string()));
418                            out.push('\n');
419                        }
420                        ErrorString(out)
421                    }
422                    LexerSrcEnvError::LexBuildErrors(errs) => {
423                        let mut out = String::new();
424                        out.push_str(&format!(
425                            "\n{ERROR}{}\n",
426                            lex_diag.file_location_msg(" building the lexer", None)
427                        ));
428                        for e in errs {
429                            out.push_str(&indent("     ", &lex_diag.format_error(e).to_string()));
430                            out.push('\n');
431                        }
432                        ErrorString(out)
433                    }
434                    e => ErrorString(e.to_string()),
435                })?;
436
437        #[cfg(test)]
438        if let Some(inspect_lexerkind_cb) = self.inspect_lexerkind_cb {
439            inspect_lexerkind_cb(build_env.lexerkind())?
440        }
441
442        let ct_parser = if let Some(ref lrcfg) = self.lrpar_config {
443            let mut closure_lexerdef = build_env.lexerdef().clone();
444            let mut ctp = CTParserBuilder::<LexerTypesT>::new().inspect_rt(Box::new(
445                move |yacc_header, rtpb, rule_ids_map, grm_path| {
446                    let owned_map = rule_ids_map
447                        .iter()
448                        .map(|(x, y)| (&**x, *y))
449                        .collect::<HashMap<_, _>>();
450                    closure_lexerdef.set_rule_ids(&owned_map);
451                    yacc_header.mark_used(&"lrpar.test_files".to_string());
452                    let grammar = rtpb.grammar();
453                    let test_glob = yacc_header.get("lrpar.test_files");
454                    let mut err_str = None;
455                    let add_error_line = |err_str: &mut Option<String>, line| {
456                        if let Some(err_str) = err_str {
457                            err_str.push_str(&format!("{}\n", line));
458                        } else {
459                            let _ = err_str.insert(format!("{}\n", line));
460                        }
461                    };
462                    match test_glob {
463                        Some(HeaderValue(_, Value::Array(test_globs, _))) => {
464                            for setting in test_globs {
465                                match setting {
466                                    Value::String(test_files, _) => {
467                                        let path_joined = grm_path.parent().unwrap().join(test_files);
468                                        let path_str = &path_joined.to_string_lossy();
469                                        let mut glob_paths = glob(path_str).map_err(|e| e.to_string())?.peekable();
470                                        if glob_paths.peek().is_none() {
471                                            return Err(format!("'test_files' glob '{}' matched no paths", path_str)
472                                                .to_string()
473                                                .into(),
474                                            );
475                                        }
476
477                                        for path in glob_paths {
478                                            let path = path?;
479                                            if let Some(ext) = path.extension()
480                                                && let Some(ext) = ext.to_str()
481                                                    && ext.starts_with("grm") {
482                                                        add_error_line(&mut err_str, "test_files extensions beginning with `grm` are reserved.".into());
483                                                    }
484                                            let input = fs::read_to_string(&path)?;
485                                            let l: LRNonStreamingLexer<LexerTypesT> =
486                                                closure_lexerdef.lexer(&input);
487                                            let errs = rtpb.parse_map(&l, &|_| (), &|_, _| ()).1;
488                                            if !errs.is_empty() {
489                                                add_error_line(&mut err_str, format!("While parsing {}:", path.display()));
490                                                for e in errs {
491                                                    let e_pp = e.pp(&l, &|t| grammar.token_epp(t));
492                                                    let e_lines = e_pp.split("\n");
493                                                    for e in e_lines {
494                                                        add_error_line(&mut err_str, format!("\t{}", e));
495                                                    }
496                                                }
497                                            }
498                                        }
499                                    }
500                                    _ => return Err("Invalid value for setting 'test_files'".into()),
501                                }
502                            }
503                            if let Some(err_str) = err_str {
504                                Err(ErrorString(err_str))?
505                            } else {
506                                Ok(())
507                            }
508
509                        }
510                        Some(_) => Err("Invalid value for setting 'test_files'".into()),
511                        None => Ok(()),
512                    }
513                },
514            ));
515            ctp = lrcfg(ctp);
516            let ct_parser = ctp.build()?;
517            self.rule_ids_map = Some(ct_parser.token_map().to_owned());
518            Some(ct_parser)
519        } else {
520            None
521        };
522
523        let unused_header_values = build_env.header().unused();
524        if !unused_header_values.is_empty() {
525            return Err(
526                format!("Unused header values: {}", unused_header_values.join(", ")).into(),
527            );
528        }
529
530        let code_gen = build_env
531            .code_generator(self.rule_ids_map, env!("VERGEN_BUILD_TIMESTAMP"))
532            .map_err(|e| match e {})?;
533        let (mut missing_from_lexer, missing_from_parser) = {
534            let lexerdef = Box::new(build_env.lexerdef_mut());
535            match code_gen.rule_ids_map() {
536                Some(rim) => {
537                    // Convert from HashMap<String, _> to HashMap<&str, _>
538                    let owned_map = rim
539                        .iter()
540                        .map(|(x, y)| (&**x, *y))
541                        .collect::<HashMap<_, _>>();
542                    let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map);
543                    (
544                        x.map(|a| a.iter().map(|&b| b.to_string()).collect::<HashSet<_>>()),
545                        y.map(|a| {
546                            a.iter()
547                                .map(|(b, span)| (b.to_string(), *span))
548                                .collect::<HashSet<_>>()
549                        }),
550                    )
551                }
552                None => (None, None),
553            }
554        };
555        let lexerdef = build_env.lexerdef();
556        if let Some(mut mfl) = missing_from_lexer.take() {
557            for tok in &lexerdef.expected_missing_tokens {
558                mfl.remove(tok.as_str());
559            }
560            if mfl.is_empty() {
561                missing_from_lexer = None;
562            } else {
563                missing_from_lexer = Some(mfl);
564            }
565        }
566
567        let mut has_unallowed_missing = false;
568        let err_indent = " ".repeat(ERROR.len());
569        if !self.allow_missing_terms_in_lexer
570            && let Some(ref mfl) = missing_from_lexer
571        {
572            if let Some(ct_parser) = &ct_parser {
573                let grm = ct_parser.yacc_grammar();
574                let token_spans = mfl
575                    .iter()
576                    .map(|name| {
577                        ct_parser
578                            .yacc_grammar()
579                            .token_span(*grm.tokens_map().get(name.as_str()).unwrap())
580                            .expect("Given token should have a span")
581                    })
582                    .collect::<Vec<_>>();
583
584                let yacc_diag = SpannedDiagnosticFormatter::new(
585                    ct_parser.grammar_src(),
586                    ct_parser.grammar_path(),
587                );
588
589                eprintln!(
590                    "{ERROR} these tokens are not referenced in the lexer but defined as follows"
591                );
592                eprintln!(
593                    "{err_indent} {}",
594                    yacc_diag.file_location_msg("in the grammar", None)
595                );
596                for span in token_spans {
597                    eprintln!(
598                        "{}",
599                        yacc_diag.underline_span_with_text(
600                            span,
601                            "Missing from lexer".to_string(),
602                            '^'
603                        )
604                    );
605                }
606                eprintln!();
607            } else {
608                eprintln!(
609                    "{ERROR} the following tokens are used in the grammar but are not defined in the lexer:"
610                );
611                for n in mfl {
612                    eprintln!("    {}", n);
613                }
614            }
615            has_unallowed_missing = true;
616        }
617        if !self.allow_missing_tokens_in_parser
618            && self.show_warnings
619            && let Some(ref mfp) = missing_from_parser
620        {
621            let error_prefix = if self.warnings_are_errors {
622                ERROR
623            } else {
624                WARNING
625            };
626            let err_indent = " ".repeat(error_prefix.len());
627            let mut outs = Vec::new();
628            outs.push(format!("{error_prefix} these tokens are not referenced in the grammar but defined as follows"));
629            outs.push(format!(
630                "{err_indent} {}",
631                lex_diag.file_location_msg("in the lexer", None)
632            ));
633            for (_, span) in mfp {
634                let error_contents = lex_diag.underline_span_with_text(
635                    *span,
636                    "Missing from parser".to_string(),
637                    '^',
638                );
639                outs.extend(error_contents.lines().map(|s| s.to_string()));
640            }
641
642            for s in outs {
643                if !self.warnings_are_errors && std::env::var("OUT_DIR").is_ok() {
644                    println!("cargo:warning={}", s)
645                } else {
646                    eprintln!("{}", s);
647                }
648            }
649
650            has_unallowed_missing |= self.warnings_are_errors;
651        }
652        if has_unallowed_missing {
653            fs::remove_file(outp).ok();
654            panic!();
655        }
656        let outs = code_gen
657            .generate(&build_env)
658            .map_err(|e| ErrorString(e.to_string()))?;
659        // If the file we're about to write out already exists with the same contents, then we
660        // don't overwrite it (since that will force a recompile of the file, and relinking of the
661        // binary etc).
662        if let Ok(curs) = read_to_string(outp)
663            && curs == outs
664        {
665            return Ok(CTLexer {
666                missing_from_lexer,
667                missing_from_parser,
668            });
669        }
670        let mut f = File::create(outp)?;
671        f.write_all(outs.as_bytes())?;
672        Ok(CTLexer {
673            missing_from_lexer,
674            missing_from_parser,
675        })
676    }
677
678    /// Given the filename `a/b.l` as input, statically compile the file `src/a/b.l` into a Rust
679    /// module which can then be imported using `lrlex_mod!("a/b.l")`. This is a convenience
680    /// function around [`process_file`](struct.CTLexerBuilder.html#method.process_file) which makes
681    /// it easier to compile `.l` files stored in a project's `src/` directory: please see
682    /// [`process_file`](#method.process_file) for additional constraints and information about the
683    /// generated files.
684    #[deprecated(
685        since = "0.11.0",
686        note = "Please use lexer_in_src_dir() and build() instead"
687    )]
688    #[allow(deprecated)]
689    pub fn process_file_in_src(
690        self,
691        srcp: &str,
692    ) -> Result<(Option<HashSet<String>>, Option<HashSet<String>>), Box<dyn Error>> {
693        let mut inp = current_dir()?;
694        inp.push("src");
695        inp.push(srcp);
696        let mut outp = PathBuf::new();
697        outp.push(var("OUT_DIR").unwrap());
698        outp.push(Path::new(srcp).parent().unwrap().to_str().unwrap());
699        create_dir_all(&outp)?;
700        let mut leaf = Path::new(srcp)
701            .file_name()
702            .unwrap()
703            .to_str()
704            .unwrap()
705            .to_owned();
706        write!(leaf, ".{}", RUST_FILE_EXT).ok();
707        outp.push(leaf);
708        self.process_file(inp, outp)
709    }
710
711    /// Statically compile the `.l` file `inp` into Rust, placing the output into the file `outp`.
712    /// The latter defines a module as follows:
713    ///
714    /// ```text
715    ///    mod modname {
716    ///      pub fn lexerdef() -> LexerDef<LexerTypesT::StorageT> { ... }
717    ///
718    ///      ...
719    ///    }
720    /// ```
721    ///
722    /// where:
723    ///  * `modname` is either:
724    ///    * the module name specified [`mod_name`](#method.mod_name)
725    ///    * or, if no module name was explicitly specified, then for the file `/a/b/c.l` the
726    ///      module name is `c_l` (i.e. the file's leaf name, minus its extension, with a prefix of
727    ///      `_l`).
728    #[deprecated(
729        since = "0.11.0",
730        note = "Please use lexer_in_src_dir() and build() instead"
731    )]
732    pub fn process_file<P, Q>(
733        mut self,
734        inp: P,
735        outp: Q,
736    ) -> Result<(Option<HashSet<String>>, Option<HashSet<String>>), Box<dyn Error>>
737    where
738        P: AsRef<Path>,
739        Q: AsRef<Path>,
740    {
741        self.lexer_path = Some(inp.as_ref().to_owned());
742        self.output_path = Some(outp.as_ref().to_owned());
743        let cl = self.build()?;
744        Ok((
745            cl.missing_from_lexer().map(|x| x.to_owned()),
746            cl.missing_from_parser()
747                .map(|x| x.iter().map(|(n, _)| n.to_owned()).collect::<HashSet<_>>()),
748        ))
749    }
750
751    /// If passed false, tokens used in the grammar but not defined in the lexer will cause a
752    /// panic at lexer generation time. Defaults to false.
753    pub fn allow_missing_terms_in_lexer(mut self, allow: bool) -> Self {
754        self.allow_missing_terms_in_lexer = allow;
755        self
756    }
757
758    /// If passed false, tokens defined in the lexer but not used in the grammar will cause a
759    /// warning at lexer generation time. Defaults to false (since lexers sometimes define tokens such
760    /// as reserved words, which are intentionally not in the grammar).
761    pub fn allow_missing_tokens_in_parser(mut self, allow: bool) -> Self {
762        self.allow_missing_tokens_in_parser = allow;
763        self
764    }
765
766    /// If set to true, [CTLexerBuilder::build] will return an error if the given lexer contains
767    /// any warnings. Defaults to `true`.
768    pub fn warnings_are_errors(mut self, flag: bool) -> Self {
769        self.warnings_are_errors = flag;
770        self
771    }
772
773    /// If set to true, [CTParserBuilder::build] will print warnings to stderr, or via cargo when
774    /// running under cargo. Defaults to `true`.
775    pub fn show_warnings(mut self, flag: bool) -> Self {
776        self.show_warnings = flag;
777        self
778    }
779
780    /// Enables `// comment` style parsing according to `flag``.
781    /// When enabled comments can appear at the beginning of a line,
782    /// and regular expressions with the `/` character should be escaped via `\/`.
783    ///
784    /// The default value is `false`.
785    ///
786    /// Setting this flag will override the same flag within a `%grmtools` section.
787    pub fn allow_wholeline_comments(mut self, flag: bool) -> Self {
788        let key = "lrlex.allow_wholeline_comments".to_string();
789        self.header.insert(
790            key,
791            HeaderValue(
792                Location::Other("CTLexerBuilder".to_string()),
793                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
794            ),
795        );
796        self
797    }
798
799    /// Sets the `regex::RegexBuilder` option of the same name.
800    /// The default value is `true`.
801    ///
802    /// Setting this flag will override the same flag within a `%grmtools` section.
803    pub fn dot_matches_new_line(mut self, flag: bool) -> Self {
804        let key = "regex.dot_matches_new_line".to_string();
805        self.header.insert(
806            key,
807            HeaderValue(
808                Location::Other("CTLexerBuilder".to_string()),
809                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
810            ),
811        );
812        self
813    }
814
815    /// Sets the `regex::RegexBuilder` option of the same name.
816    /// The default value is `true`.
817    ///
818    /// Setting this flag will override the same flag within a `%grmtools` section.
819    pub fn multi_line(mut self, flag: bool) -> Self {
820        let key = "regex.multi_line".to_string();
821        self.header.insert(
822            key,
823            HeaderValue(
824                Location::Other("CTLexerBuilder".to_string()),
825                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
826            ),
827        );
828        self
829    }
830
831    /// Enables posix lex compatible escape sequences according to `flag`.
832    /// The default value is `false`.
833    ///
834    /// Setting this flag will override the same flag within a `%grmtools` section.
835    pub fn posix_escapes(mut self, flag: bool) -> Self {
836        let key = "lrlex.posix_escapes".to_string();
837        self.header.insert(
838            key,
839            HeaderValue(
840                Location::Other("CTLexerBuilder".to_string()),
841                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
842            ),
843        );
844        self
845    }
846
847    /// Sets the `regex::RegexBuilder` option of the same name.
848    /// The default value is `true`.
849    ///
850    /// Setting this flag will override the same flag within a `%grmtools` section.
851    pub fn octal(mut self, flag: bool) -> Self {
852        let key = "regex.octal".to_string();
853        self.header.insert(
854            key,
855            HeaderValue(
856                Location::Other("CTLexerBuilder".to_string()),
857                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
858            ),
859        );
860        self
861    }
862
863    /// Sets the `regex::RegexBuilder` option of the same name.
864    /// Default value is specified by regex.
865    ///
866    /// Setting this flag will override the same flag within a `%grmtools` section.
867    pub fn swap_greed(mut self, flag: bool) -> Self {
868        let key = "regex.swap_greed".to_string();
869        self.header.insert(
870            key,
871            HeaderValue(
872                Location::Other("CTLexerBuilder".to_string()),
873                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
874            ),
875        );
876        self
877    }
878
879    /// Sets the `regex::RegexBuilder` option of the same name.
880    /// Default value is specified by regex.
881    ///
882    /// Setting this flag will override the same flag within a `%grmtools` section.
883    pub fn ignore_whitespace(mut self, flag: bool) -> Self {
884        let key = "regex.ignore_whitespace".to_string();
885        self.header.insert(
886            key,
887            HeaderValue(
888                Location::Other("CTLexerBuilder".to_string()),
889                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
890            ),
891        );
892        self
893    }
894
895    /// Sets the `regex::RegexBuilder` option of the same name.
896    /// Default value is specified by regex.
897    ///
898    /// Setting this flag will override the same flag within a `%grmtools` section.
899    pub fn unicode(mut self, flag: bool) -> Self {
900        let key = "regex.unicode".to_string();
901        self.header.insert(
902            key,
903            HeaderValue(
904                Location::Other("CTLexerBuilder".to_string()),
905                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
906            ),
907        );
908        self
909    }
910
911    /// Sets the `regex::RegexBuilder` option of the same name.
912    /// Default value is specified by regex.
913    ///
914    /// Setting this flag will override the same flag within a `%grmtools` section.
915    pub fn case_insensitive(mut self, flag: bool) -> Self {
916        let key = "regex.case_insensitive".to_string();
917        self.header.insert(
918            key,
919            HeaderValue(
920                Location::Other("CTLexerBuilder".to_string()),
921                Value::Bool(flag, Location::Other("CTLexerBuilder".to_string())),
922            ),
923        );
924        self
925    }
926
927    /// Sets the `regex::RegexBuilder` option of the same name.
928    /// Default value is specified by regex.
929    ///
930    /// Setting this flag will override the same flag within a `%grmtools` section.
931    pub fn size_limit(mut self, sz: usize) -> Self {
932        let key = "regex.size_limit".to_string();
933        self.header.insert(
934            key,
935            HeaderValue(
936                Location::Other("CTLexerBuilder".to_string()),
937                Value::Num(sz as u64, Location::Other("CTLexerBuilder".to_string())),
938            ),
939        );
940        self
941    }
942
943    /// Sets the `regex::RegexBuilder` option of the same name.
944    /// Default value is specified by regex.
945    ///
946    /// Setting this flag will override the same flag within a `%grmtools` section.
947    pub fn dfa_size_limit(mut self, sz: usize) -> Self {
948        let key = "regex.dfa_size_limit".to_string();
949        self.header.insert(
950            key,
951            HeaderValue(
952                Location::Other("CTLexerBuilder".to_string()),
953                Value::Num(sz as u64, Location::Other("CTLexerBuilder".to_string())),
954            ),
955        );
956        self
957    }
958
959    /// Sets the `regex::RegexBuilder` option of the same name.
960    /// Default value is specified by regex.
961    ///
962    /// Setting this flag will override the same flag within a `%grmtools` section.
963    pub fn nest_limit(mut self, lim: u32) -> Self {
964        let key = "regex.nest_limit".to_string();
965        self.header.insert(
966            key,
967            HeaderValue(
968                Location::Other("CTLexerBuilder".to_string()),
969                Value::Num(lim as u64, Location::Other("CTLexerBuilder".to_string())),
970            ),
971        );
972        self
973    }
974
975    #[cfg(test)]
976    pub fn inspect_lexerkind(
977        mut self,
978        cb: Box<dyn Fn(&LexerKind) -> Result<(), Box<dyn Error>>>,
979    ) -> Self {
980        self.inspect_lexerkind_cb = Some(cb);
981        self
982    }
983}
984
985/// An interface to the result of [CTLexerBuilder::build()].
986pub struct CTLexer {
987    missing_from_lexer: Option<HashSet<String>>,
988    missing_from_parser: Option<HashSet<(String, Span)>>,
989}
990
991impl CTLexer {
992    fn missing_from_lexer(&self) -> Option<&HashSet<String>> {
993        self.missing_from_lexer.as_ref()
994    }
995
996    fn missing_from_parser(&self) -> Option<&HashSet<(String, Span)>> {
997        self.missing_from_parser.as_ref()
998    }
999}
1000
1001/// Exports all token IDs used by a parser as a separate Rust module.
1002///
1003/// This builder will create a Rust module named `mod_name`
1004/// that can be imported with [`lrlex_mod!(mod_name)`](crate::lrlex_mod).
1005/// The module will contain one `const` `StorageT` per token in `token_map`,
1006/// with the token prefixed by `T_`. In addition, it will contain
1007/// an array of all token IDs `TOK_IDS`.
1008///
1009/// For example, if `StorageT` is `u8`, `mod_name` is `x`, and `token_map` is
1010/// `HashMap{"ID": 0, "INT": 1}` the generated module will look roughly as follows:
1011///
1012/// ```rust,ignore
1013/// mod x {
1014///   pub const T_ID: u8 = 0;
1015///   pub const T_INT: u8 = 1;
1016///   pub const TOK_IDS: &[u8] = &[T_ID, T_INT];
1017/// }
1018/// ```
1019///
1020/// See the [custom lexer example] for more usage details.
1021///
1022/// [custom lexer example]: https://github.com/softdevteam/grmtools/tree/master/lrlex/examples/calc_manual_lex
1023#[derive(Debug, Clone)]
1024pub struct CTTokenMapBuilder<StorageT: Display + ToTokens> {
1025    mod_name: String,
1026    token_map: Vec<(String, TokenStream)>,
1027    rename_map: Option<HashMap<String, String>>,
1028    allow_dead_code: bool,
1029    _marker: PhantomData<StorageT>,
1030}
1031
1032impl<StorageT: Display + ToTokens> CTTokenMapBuilder<StorageT> {
1033    /// Create a new token map builder.
1034    ///
1035    /// See the [builder documentation] for more info.
1036    ///
1037    /// [builder documentation]: CTTokenMapBuilder
1038    pub fn new(
1039        mod_name: impl Into<String>,
1040        token_map: impl Borrow<HashMap<String, StorageT>>,
1041    ) -> Self {
1042        Self {
1043            mod_name: mod_name.into(),
1044            token_map: token_map
1045                .borrow()
1046                .iter()
1047                .map(|(tok_name, tok_value)| (tok_name.clone(), tok_value.to_token_stream()))
1048                .collect(),
1049            rename_map: None,
1050            allow_dead_code: false,
1051            _marker: PhantomData,
1052        }
1053    }
1054
1055    /// Set a token rename map.
1056    ///
1057    /// Rename map is used to specify identifier names for tokens whose names
1058    /// are not valid Rust identifiers. For example, if `token_map`
1059    /// is `HashMap{"+": 0, "ID": 1}` and `rename_map` is `HashMap{"+": "PLUS"}`
1060    /// then the generated module will look roughly as follows:
1061    ///
1062    /// ```rust,ignore
1063    /// mod x {
1064    ///   pub const T_PLUS: u8 = 0;
1065    ///   pub const T_ID: u8 = 1;
1066    /// }
1067    /// ```
1068    pub fn rename_map<M, I, K, V>(mut self, rename_map: Option<M>) -> Self
1069    where
1070        M: IntoIterator<Item = I>,
1071        I: Borrow<(K, V)>,
1072        K: AsRef<str>,
1073        V: AsRef<str>,
1074    {
1075        self.rename_map = rename_map.map(|rename_map| {
1076            rename_map
1077                .into_iter()
1078                .map(|it| {
1079                    let (k, v) = it.borrow();
1080                    let k = k.as_ref().into();
1081                    let v = v.as_ref().into();
1082                    (k, v)
1083                })
1084                .collect()
1085        });
1086        self
1087    }
1088
1089    /// Control whether the builder will add `#[allow(dead_code)]`
1090    /// to the generated module.
1091    ///
1092    /// By default, all tokens are `#[deny(dead_code)]`, meaning that you'll
1093    /// get a warning if your custom lexer doesn't use any of them.
1094    /// This function can be used to disable this behavior.
1095    pub fn allow_dead_code(mut self, allow_dead_code: bool) -> Self {
1096        self.allow_dead_code = allow_dead_code;
1097        self
1098    }
1099
1100    /// Build the token map module.
1101    pub fn build(&self) -> Result<(), Box<dyn Error>> {
1102        // Record the time that this version of lrlex was built. If the source code changes and rustc
1103        // forces a recompile, this will change this value, causing anything which depends on this
1104        // build of lrlex to be recompiled too.
1105        let mut outs = String::new();
1106        let timestamp = env!("VERGEN_BUILD_TIMESTAMP");
1107        let mod_ident = format_ident!("{}", self.mod_name);
1108        write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok();
1109        let storaget = str::parse::<TokenStream>(type_name::<StorageT>()).unwrap();
1110        // Sort the tokens so that they're always in the same order.
1111        // This will prevent unneeded rebuilds.
1112        let mut token_map_sorted = self.token_map.clone();
1113        token_map_sorted.sort_by(|(l, _), (r, _)| l.cmp(r));
1114        let (token_array, tokens) = token_map_sorted
1115            .iter()
1116            .map(|(k, id)| {
1117                let name = match &self.rename_map {
1118                    Some(rmap) => rmap.get(k).unwrap_or(k),
1119                    _ => k,
1120                };
1121                let tok_ident: Ident = syn::parse_str(&format!("T_{}", name.to_ascii_uppercase()))
1122                    .map_err(|e| {
1123                        format!(
1124                            "token name {:?} is not a valid Rust identifier: {}; \
1125                            consider renaming it via `CTTokenMapBuilder::rename_map`.",
1126                            name, e
1127                        )
1128                    })?;
1129                Ok((
1130                    // Note: the array of all tokens can't use `tok_ident` because
1131                    // it will confuse the dead code checker. For this reason,
1132                    // we use `id` here.
1133                    quote! {
1134                        #id,
1135                    },
1136                    quote! {
1137                        pub const #tok_ident: #storaget = #id;
1138                    },
1139                ))
1140            })
1141            .collect::<Result<(TokenStream, TokenStream), Box<dyn Error>>>()?;
1142        let unused_annotation = if self.allow_dead_code {
1143            quote! {#[allow(dead_code)]}
1144        } else {
1145            quote! {}
1146        };
1147        // Since the formatter doesn't preserve comments and we don't want to lose build time,
1148        // just format the module contents.
1149        let unformatted = quote! {
1150            #unused_annotation
1151            mod #mod_ident {
1152                #tokens
1153                #[allow(dead_code)]
1154                pub const TOK_IDS: &[#storaget] = &[#token_array];
1155            }
1156        }
1157        .to_string();
1158        let out_mod = syn::parse_str(&unformatted)
1159            .map(|syntax_tree| prettyplease::unparse(&syntax_tree))
1160            .unwrap_or(unformatted);
1161        outs.push_str(&out_mod);
1162        let mut outp = PathBuf::from(var("OUT_DIR")?);
1163        outp.push(&self.mod_name);
1164        outp.set_extension("rs");
1165
1166        // If the file we're about to write out already exists with the same contents, then we
1167        // don't overwrite it (since that will force a recompile of the file, and relinking of the
1168        // binary etc).
1169        if let Ok(curs) = read_to_string(&outp)
1170            && curs == outs
1171        {
1172            return Ok(());
1173        }
1174
1175        let mut f = File::create(outp)?;
1176        f.write_all(outs.as_bytes())?;
1177        Ok(())
1178    }
1179}
1180
1181/// Create a Rust module named `mod_name` that can be imported with
1182/// [`lrlex_mod!(mod_name)`](crate::lrlex_mod).
1183///
1184/// This function is deprecated in favour of [`CTTokenMapBuilder`].
1185#[deprecated(since = "0.14.0", note = "use `lrlex::CTTokenMapBuilder` instead")]
1186pub fn ct_token_map<StorageT: Display + ToTokens>(
1187    mod_name: &str,
1188    token_map: impl Borrow<HashMap<String, StorageT>>,
1189    rename_map: Option<&HashMap<&str, &str>>,
1190) -> Result<(), Box<dyn Error>> {
1191    CTTokenMapBuilder::new(mod_name, token_map)
1192        .rename_map(rename_map)
1193        .allow_dead_code(true)
1194        .build()
1195}
1196
1197/// Indents a multi-line string and trims any trailing newline.
1198/// This currently assumes that indentation on blank lines does not matter.
1199///
1200/// The algorithm used by this function is:
1201/// 1. Prefix `s` with the indentation, indenting the first line.
1202/// 2. Trim any trailing newlines.
1203/// 3. Replace all newlines with `\n{indent}`` to indent all lines after the first.
1204///
1205/// It is plausible that we should a step 4, but currently do not:
1206/// 4. Replace all `\n{indent}\n` with `\n\n`
1207fn indent(indent: &str, s: &str) -> String {
1208    format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent))
1209}
1210
1211// It isn't clear to me why this test isn't working on wasm32,
1212// as the `workspace_runner` should allow access to `OUT_DIR`
1213// perhaps it is related to absolute paths
1214#[cfg(all(not(target_arch = "wasm32"), test))]
1215mod test {
1216    use std::fs::File;
1217    use std::io::Write;
1218
1219    use super::{CTLexerBuilder, LexerKind};
1220    #[test]
1221    fn test_grmtools_section_lexerkind() {
1222        let lexerkinds = ["LRNonStreamingLexer", "LexerKind::LRNonStreamingLexer"];
1223        for (i, kind) in lexerkinds.iter().enumerate() {
1224            let lex_src = format!(
1225                "
1226%grmtools{{lexerkind: {}}}
1227%%
1228. ;
1229",
1230                kind
1231            );
1232            let lex_path = format!(
1233                "{}/test_grmtools_section_lexerkind_{}.l",
1234                env!("OUT_DIR"),
1235                i
1236            );
1237            let mut l_file = File::create(lex_path.clone()).unwrap();
1238            l_file.write_all(lex_src.as_bytes()).unwrap();
1239            CTLexerBuilder::new()
1240                .output_path(format!("{}.rs", lex_path.clone()))
1241                .lexer_path(lex_path.clone())
1242                .inspect_lexerkind(Box::new(move |lexerkind| {
1243                    assert!(matches!(lexerkind, &LexerKind::LRNonStreamingLexer));
1244                    Ok(())
1245                }))
1246                .build()
1247                .unwrap();
1248        }
1249    }
1250
1251    #[test]
1252    /// Tests a yacc .y filename containing a dash character leading to an invalid rust identifier
1253    /// when that dash is subsequently used as the default `CTParserBuilder::mod_name`.
1254    fn test_invalid_identifier_in_derived_mod_name() {
1255        let mut lex_path = std::path::PathBuf::from(env!("OUT_DIR"));
1256        lex_path.push("contains-a-dash.l");
1257        let mut f = File::create(&lex_path).unwrap();
1258        let _ = f.write_all(
1259            r#"
1260%%
1261A  "A"
1262"#
1263            .as_bytes(),
1264        );
1265        match CTLexerBuilder::new()
1266            .output_path(format!("{}.rs", lex_path.display()))
1267            .lexer_path(lex_path.clone())
1268            .build()
1269        {
1270            Ok(_) => panic!("Expected error"),
1271            Err(e) => {
1272                let err_string = e.to_string();
1273                assert_eq!(
1274                    err_string,
1275                    "mod_name 'contains-a-dash_l' is not a valid rust identifier due to 'unexpected token'"
1276                );
1277            }
1278        }
1279    }
1280}
lrlex/ctbuilder.rs

lrlex/
ctbuilder.rs
