From 9918693037dce8aa4bb6f08741b6812923486c18 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 19 Jun 2024 11:26:03 +0200 Subject: Merging upstream version 1.76.0+dfsg1. Signed-off-by: Daniel Baumann --- vendor/pest_generator/src/generator.rs | 145 +++++++++++++++++++++++++++------ vendor/pest_generator/src/lib.rs | 50 ++++++++++-- 2 files changed, 159 insertions(+), 36 deletions(-) (limited to 'vendor/pest_generator/src') diff --git a/vendor/pest_generator/src/generator.rs b/vendor/pest_generator/src/generator.rs index e36e9eb87..d301e43a2 100644 --- a/vendor/pest_generator/src/generator.rs +++ b/vendor/pest_generator/src/generator.rs @@ -11,17 +11,17 @@ use std::path::PathBuf; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt}; -use syn::{self, Generics, Ident}; +use syn::{self, Ident}; use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; use crate::docs::DocComment; +use crate::ParsedDerive; pub(crate) fn generate( - name: Ident, - generics: &Generics, + parsed_derive: ParsedDerive, paths: Vec, rules: Vec, defaults: Vec<&str>, @@ -29,14 +29,14 @@ pub(crate) fn generate( include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); - + let name = parsed_derive.name; let builtins = generate_builtin_rules(); let include_fix = if include_grammar { generate_include(&name, paths) } else { quote!() }; - let rule_enum = generate_enum(&rules, doc_comment, uses_eoi); + let rule_enum = generate_enum(&rules, doc_comment, uses_eoi, parsed_derive.non_exhaustive); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -49,7 +49,7 @@ pub(crate) fn generate( } })); - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let (impl_generics, ty_generics, where_clause) = parsed_derive.generics.split_for_impl(); let result = result_type(); @@ -197,8 +197,13 @@ fn generate_include(name: &Ident, paths: Vec) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream { - let rules = rules.iter().map(|rule| { +fn generate_enum( + rules: &[OptimizedRule], + doc_comment: &DocComment, + uses_eoi: bool, + non_exhaustive: bool, +) -> TokenStream { + let rule_variants = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); match doc_comment.line_docs.get(&rule.name) { @@ -213,26 +218,49 @@ fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bo }); let grammar_doc = &doc_comment.grammar_doc; + let mut result = quote! { + #[doc = #grammar_doc] + #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + }; + if non_exhaustive { + result.append_all(quote! { + #[non_exhaustive] + }); + } + result.append_all(quote! { + pub enum Rule + }); if uses_eoi { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { + result.append_all(quote! { + { + #[doc = "End-of-input"] EOI, - #( #rules ),* + #( #rule_variants ),* } - } + }); } else { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { - #( #rules ),* + result.append_all(quote! { + { + #( #rule_variants ),* + } + }) + }; + + let rules = rules.iter().map(|rule| { + let rule_name = format_ident!("r#{}", rule.name); + quote! { #rule_name } + }); + + result.append_all(quote! { + impl Rule { + pub fn all_rules() -> &'static[Rule] { + &[ #(Rule::#rules), * ] } } - } + }); + + result } fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { @@ -496,6 +524,26 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream { }) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::RepOnce(expr) => { + let expr = generate_expr(*expr); + + quote! { + state.sequence(|state| { + #expr.and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + super::hidden::skip( + state + ).and_then(|state| { + #expr + }) + }) + }) + }) + }) + } + } OptimizedExpr::Skip(strings) => { quote! { let strings = [#(#strings),*]; @@ -520,8 +568,14 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream { #[cfg(feature = "grammar-extras")] OptimizedExpr::NodeTag(expr, tag) => { let expr = generate_expr(*expr); + let tag_cow = { + #[cfg(feature = "std")] + quote! { ::std::borrow::Cow::Borrowed(#tag) } + #[cfg(not(feature = "std"))] + quote! { ::alloc::borrow::Cow::Borrowed(#tag) } + }; quote! { - #expr.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed(#tag))) + #expr.and_then(|state| state.tag_node(#tag_cow)) } } } @@ -635,6 +689,22 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream { }) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::RepOnce(expr) => { + let expr = generate_expr_atomic(*expr); + + quote! { + state.sequence(|state| { + #expr.and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + #expr + }) + }) + }) + }) + } + } OptimizedExpr::Skip(strings) => { quote! { let strings = [#(#strings),*]; @@ -659,8 +729,14 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream { #[cfg(feature = "grammar-extras")] OptimizedExpr::NodeTag(expr, tag) => { let expr = generate_expr_atomic(*expr); + let tag_cow = { + #[cfg(feature = "std")] + quote! { ::std::borrow::Cow::Borrowed(#tag) } + #[cfg(not(feature = "std"))] + quote! { ::alloc::borrow::Cow::Borrowed(#tag) } + }; quote! { - #expr.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed(#tag))) + #expr.and_then(|state| state.tag_node(#tag_cow)) } } } @@ -708,6 +784,7 @@ mod tests { use proc_macro2::Span; use std::collections::HashMap; + use syn::Generics; #[test] fn rule_enum_simple() { @@ -726,7 +803,7 @@ mod tests { }; assert_eq!( - generate_enum(&rules, doc_comment, false).to_string(), + generate_enum(&rules, doc_comment, false, false).to_string(), quote! { #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] @@ -735,6 +812,11 @@ mod tests { #[doc = "This is rule comment"] r#f } + impl Rule { + pub fn all_rules() -> &'static [Rule] { + &[Rule::r#f] + } + } } .to_string() ); @@ -1047,9 +1129,13 @@ mod tests { let base_path = current_dir.join("base.pest").to_str().unwrap().to_string(); let test_path = current_dir.join("test.pest").to_str().unwrap().to_string(); - + let parsed_derive = ParsedDerive { + name, + generics, + non_exhaustive: false, + }; assert_eq!( - generate(name, &generics, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), + generate(parsed_derive, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: [&'static str; 2usize] = [include_str!(#base_path), include_str!(#test_path)]; @@ -1062,6 +1148,11 @@ mod tests { #[doc = "If statement"] r#if } + impl Rule { + pub fn all_rules() -> &'static [Rule] { + &[Rule::r#a, Rule::r#if] + } + } #[allow(clippy::all)] impl ::pest::Parser for MyParser { diff --git a/vendor/pest_generator/src/lib.rs b/vendor/pest_generator/src/lib.rs index 98c726525..cbd13eaf5 100644 --- a/vendor/pest_generator/src/lib.rs +++ b/vendor/pest_generator/src/lib.rs @@ -42,7 +42,7 @@ use pest_meta::{optimizer, unwrap_or_report, validator}; /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let ast: DeriveInput = syn::parse2(input).unwrap(); - let (name, generics, contents) = parse_derive(ast); + let (parsed_derive, contents) = parse_derive(ast); let mut data = String::new(); let mut paths = vec![]; @@ -97,8 +97,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let optimized = optimizer::optimize(ast); generator::generate( - name, - &generics, + parsed_derive, paths, optimized, defaults, @@ -120,7 +119,13 @@ enum GrammarSource { Inline(String), } -fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec) { +struct ParsedDerive { + pub(crate) name: Ident, + pub(crate) generics: Generics, + pub(crate) non_exhaustive: bool, +} + +fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { let name = ast.ident; let generics = ast.generics; @@ -142,7 +147,19 @@ fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec) { grammar_sources.push(get_attribute(attr)) } - (name, generics, grammar_sources) + let non_exhaustive = ast + .attrs + .iter() + .any(|attr| attr.meta.path().is_ident("non_exhaustive")); + + ( + ParsedDerive { + name, + generics, + non_exhaustive, + }, + grammar_sources, + ) } fn get_attribute(attr: &Attribute) -> GrammarSource { @@ -177,7 +194,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]); } @@ -189,8 +206,9 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (parsed_derive, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(!parsed_derive.non_exhaustive); } #[test] @@ -202,7 +220,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!( filenames, [ @@ -212,6 +230,19 @@ mod tests { ); } + #[test] + fn derive_nonexhaustive() { + let definition = " + #[non_exhaustive] + #[grammar = \"myfile.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (parsed_derive, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(parsed_derive.non_exhaustive); + } + #[test] #[should_panic(expected = "grammar attribute must be a string")] fn derive_wrong_arg() { @@ -242,6 +273,7 @@ mod tests { fn test_generate_doc() { let input = quote! { #[derive(Parser)] + #[non_exhaustive] #[grammar = "../tests/test.pest"] pub struct TestParser; }; @@ -252,7 +284,7 @@ mod tests { #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space\n"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - + #[non_exhaustive] pub enum Rule { #[doc = "Matches foo str, e.g.: `foo`"] r#foo, -- cgit v1.2.3