lrlex/
mod.rs

1//! `lrlex` is a partial replacement for [`lex`](http://dinosaur.compilertools.net/lex/index.html)
2//! / [`flex`](https://westes.github.io/flex/manual/). It takes in a `.l` file and statically
3//! compiles it to Rust code. The resulting [LRNonStreamingLexerDef] can then be given an input
4//! string, from which it instantiates an [LRNonStreamingLexer]. This provides an iterator which
5//! can produce the sequence of [lrpar::Lexeme]s for that input, as well as answer basic queries
6//! about [cfgrammar::Span]s (e.g. extracting substrings, calculating line and column numbers).
7
8#![allow(clippy::new_without_default)]
9#![allow(clippy::type_complexity)]
10#![allow(clippy::unnecessary_wraps)]
11#![allow(clippy::upper_case_acronyms)]
12#![forbid(unsafe_code)]
13#![deny(unreachable_pub)]
14
15use std::{error::Error, fmt};
16
17mod codegen;
18mod ctbuilder;
19#[doc(hidden)]
20pub mod defaults;
21mod lexer;
22mod parser;
23
24#[allow(deprecated)]
25pub use crate::{
26    ctbuilder::{
27        CTLexer, CTLexerBuilder, CTTokenMapBuilder, LexerKind, RustEdition, Visibility,
28        ct_token_map,
29    },
30    defaults::{DefaultLexeme, DefaultLexerTypes},
31    lexer::{
32        DEFAULT_LEX_FLAGS, LRNonStreamingLexer, LRNonStreamingLexerDef, LexFlags, LexerDef, Rule,
33        UNSPECIFIED_LEX_FLAGS,
34    },
35    parser::StartState,
36    parser::StartStateOperation,
37};
38
39use cfgrammar::header::{HeaderError, HeaderErrorKind};
40use cfgrammar::yacc::parser::SpansKind;
41use cfgrammar::{Span, Spanned};
42
43pub type LexBuildResult<T> = Result<T, Vec<LexBuildError>>;
44
45/// Any error from the Lex parser returns an instance of this struct.
46#[derive(Debug)]
47pub struct LexBuildError {
48    pub(crate) kind: LexErrorKind,
49    pub(crate) spans: Vec<Span>,
50}
51
52impl Error for LexBuildError {}
53
54/// The various different possible Lex parser errors.
55#[derive(Debug, Clone)]
56#[non_exhaustive]
57pub enum LexErrorKind {
58    PrematureEnd,
59    RoutinesNotSupported,
60    UnknownDeclaration,
61    MissingSpace,
62    InvalidName,
63    UnknownStartState,
64    DuplicateStartState,
65    InvalidStartState,
66    InvalidStartStateName,
67    DuplicateName,
68    RegexError(regex::Error),
69    VerbatimNotSupported,
70    Header(HeaderErrorKind, SpansKind),
71}
72
73impl LexErrorKind {
74    fn is_same_kind(&self, other: &Self) -> bool {
75        use LexErrorKind as EK;
76        matches!(
77            (self, other),
78            (EK::PrematureEnd, EK::PrematureEnd)
79                | (EK::RoutinesNotSupported, EK::RoutinesNotSupported)
80                | (EK::UnknownDeclaration, EK::UnknownDeclaration)
81                | (EK::MissingSpace, EK::MissingSpace)
82                | (EK::InvalidName, EK::InvalidName)
83                | (EK::UnknownStartState, EK::UnknownStartState)
84                | (EK::DuplicateStartState, EK::DuplicateStartState)
85                | (EK::InvalidStartState, EK::InvalidStartState)
86                | (EK::InvalidStartStateName, EK::InvalidStartStateName)
87                | (EK::DuplicateName, EK::DuplicateName)
88                | (EK::RegexError(_), EK::RegexError(_))
89                | (EK::VerbatimNotSupported, EK::VerbatimNotSupported)
90        )
91    }
92}
93
94impl Spanned for LexBuildError {
95    fn spans(&self) -> &[Span] {
96        self.spans.as_slice()
97    }
98
99    fn spanskind(&self) -> SpansKind {
100        match self.kind {
101            LexErrorKind::PrematureEnd
102            | LexErrorKind::RoutinesNotSupported
103            | LexErrorKind::UnknownDeclaration
104            | LexErrorKind::MissingSpace
105            | LexErrorKind::InvalidName
106            | LexErrorKind::UnknownStartState
107            | LexErrorKind::InvalidStartState
108            | LexErrorKind::InvalidStartStateName
109            | LexErrorKind::VerbatimNotSupported
110            | LexErrorKind::RegexError(_) => SpansKind::Error,
111            LexErrorKind::DuplicateName | LexErrorKind::DuplicateStartState => {
112                SpansKind::DuplicationError
113            }
114            LexErrorKind::Header(_, spanskind) => spanskind,
115        }
116    }
117}
118
119impl fmt::Display for LexBuildError {
120    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
121        let s = match &self.kind {
122            LexErrorKind::VerbatimNotSupported => "Verbatim code not supported",
123            LexErrorKind::PrematureEnd => "File ends prematurely",
124            LexErrorKind::RoutinesNotSupported => "Routines not currently supported",
125            LexErrorKind::UnknownDeclaration => "Unknown declaration",
126            LexErrorKind::MissingSpace => "Rule is missing a space",
127            LexErrorKind::InvalidName => "Invalid rule name",
128            LexErrorKind::UnknownStartState => "Start state not known",
129            LexErrorKind::DuplicateStartState => "Start state already exists",
130            LexErrorKind::InvalidStartState => "Invalid start state",
131            LexErrorKind::InvalidStartStateName => "Invalid start state name",
132            LexErrorKind::DuplicateName => "Rule name already exists",
133            LexErrorKind::RegexError(e) => return write!(f, "Invalid regular expression: {e}"),
134            LexErrorKind::Header(e, _) => return write!(f, "In '%grmtools' section {e}"),
135        };
136        write!(f, "{s}")
137    }
138}
139
140impl From<HeaderError<Span>> for LexBuildError {
141    fn from(e: HeaderError<Span>) -> LexBuildError {
142        LexBuildError {
143            kind: LexErrorKind::Header(e.kind, e.spanskind()),
144            spans: e.locations,
145        }
146    }
147}
148
149#[derive(Copy, Clone, Debug)]
150pub struct StartStateId {
151    _id: usize,
152}
153
154impl StartStateId {
155    fn new(id: usize) -> Self {
156        Self { _id: id }
157    }
158}
159
160/// A Lexing error.
161#[derive(Clone, Debug)]
162pub struct LRLexError {
163    span: Span,
164    lexing_state: Option<StartStateId>,
165}
166
167impl lrpar::LexError for LRLexError {
168    fn span(&self) -> Span {
169        self.span
170    }
171}
172
173impl LRLexError {
174    /// Construct a new LRLex error covering `span`.
175    pub fn new(span: Span) -> Self {
176        LRLexError {
177            span,
178            lexing_state: None,
179        }
180    }
181
182    /// Construct a new LRLex error covering `span` for `lexing_state`.
183    pub fn new_with_lexing_state(span: Span, lexing_state: StartStateId) -> Self {
184        LRLexError {
185            span,
186            lexing_state: Some(lexing_state),
187        }
188    }
189
190    /// Returns the state, if there was one, that the lexer was in when the error was detected.
191    pub fn lexing_state(&self) -> Option<StartStateId> {
192        self.lexing_state
193    }
194}
195
196impl Error for LRLexError {}
197
198impl fmt::Display for LRLexError {
199    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
200        write!(
201            f,
202            "Couldn't lex input starting at byte {}",
203            self.span.start()
204        )
205    }
206}
207
208#[deprecated(
209    since = "0.8.0",
210    note = "This struct has been renamed to LRNonStreamingLexerDef"
211)]
212pub type NonStreamingLexerDef<StorageT> = LRNonStreamingLexerDef<StorageT>;
213
214/// A convenience macro for including statically compiled `.l` files. A file `src/a/b/c.l`
215/// processed by [CTLexerBuilder::lexer_in_src_dir] can then be used in a crate with
216/// `lrlex_mod!("a/b/c.l")`.
217///
218/// Note that you can use `lrlex_mod` with [CTLexerBuilder::output_path] if, and only if, the
219/// output file was placed in [std::env::var]`("OUT_DIR")` or one of its subdirectories.
220#[macro_export]
221macro_rules! lrlex_mod {
222    ($path:expr) => {
223        include!(concat!(env!("OUT_DIR"), "/", $path, ".rs"));
224    };
225}
226
227/// This private module with pub items which is directly related to
228/// the "Sealed trait" pattern. These items are used within the current
229/// crate. See `unstable_api` module for enabling usage outside the crate.
230mod unstable {
231    #![allow(unused)]
232    #![allow(unreachable_pub)]
233    pub struct UnstableApi;
234    pub trait UnstableTrait {}
235}
236
237/// A module for lifting restrictions on visibility by enabling unstable features.
238///
239/// See the sources for a complete list of features, and members.
240pub mod unstable_api {
241    /// Unstable functions that take a value `UnstableApi` require
242    /// the "_unstable_api" feature. This feature controls
243    /// whether the value has `pub` visibility outside the crate.
244    #[cfg(feature = "_unstable_api")]
245    pub use crate::unstable::UnstableApi;
246
247    /// This is a a supertrait for traits that are considered to be Unstable.
248    /// Unstable traits do not provide any semver guarantees.
249    ///
250    /// Enabling the `_unsealed_unstable traits` makes this supertrait publicly
251    /// Visible.
252    ///
253    ///
254    /// Declaring an unstable Api within the crate:
255    /// ```ignore_rust
256    /// // Within the crate use `crate::unstable::` .
257    /// pub trait Foo: crate::unstable::UnstableTrait {
258    ///     fn foo(key: crate::unstable::UnstableApi);
259    /// }
260    /// ```
261    ///
262    /// Deriving the trait outside the crate (requires feature `_unsealed_unstable_traits`)
263    /// ```ignore_rust
264    /// struct Bar;
265    /// impl unstable_api::UnstableTrait for Bar{}
266    /// impl Foo for Bar {
267    ///   fn foo(key: unstable_api::UnstableApi) {
268    ///     ...
269    ///   }
270    /// }
271    /// ```
272    ///
273    ///
274    /// Calling an implementation of the trait outside the crate (requires feature `_unstable_api`:
275    /// ```ignore_rust
276    ///   let x: &dyn Foo = ...;
277    ///   x.foo(unstable_api::UnstableApi);
278    /// ```
279    #[cfg(feature = "_unsealed_unstable_traits")]
280    pub use crate::unstable::UnstableTrait;
281
282    /// An value that acts as a key to inform callers that they are
283    /// calling an unstable internal api. This value is public by default.
284    /// Access to it does not require any features to be enabled.
285    ///
286    /// Q. When this should be used?
287    ///
288    /// A. When generated code needs to call internal api within it,
289    /// where you do not want the caller to have to enable any features
290    /// to use the generated code.
291    pub struct InternalPublicApi;
292}
lrlex/mod.rs

lrlex/
mod.rs
