diff options
Diffstat (limited to 'src/tools/rust-analyzer/crates/hir-expand')
13 files changed, 566 insertions, 142 deletions
diff --git a/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml b/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml index 77eb1fd45..5c684be03 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml +++ b/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml @@ -2,9 +2,11 @@ name = "hir-expand" version = "0.0.0" description = "TBD" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" + +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true [lib] doctest = false @@ -19,16 +21,18 @@ itertools = "0.10.5" hashbrown = { version = "0.12.1", features = [ "inline-more", ], default-features = false } -smallvec = { version = "1.10.0", features = ["const_new"] } +smallvec.workspace = true -stdx = { path = "../stdx", version = "0.0.0" } -base-db = { path = "../base-db", version = "0.0.0" } -cfg = { path = "../cfg", version = "0.0.0" } -syntax = { path = "../syntax", version = "0.0.0" } -profile = { path = "../profile", version = "0.0.0" } -tt = { path = "../tt", version = "0.0.0" } -mbe = { path = "../mbe", version = "0.0.0" } -limit = { path = "../limit", version = "0.0.0" } +# local deps +stdx.workspace = true +intern.workspace = true +base-db.workspace = true +cfg.workspace = true +syntax.workspace = true +profile.workspace = true +tt.workspace = true +mbe.workspace = true +limit.workspace = true [dev-dependencies] expect-test = "1.4.0" diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs new file mode 100644 index 000000000..5c04f8e8b --- /dev/null +++ b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs @@ -0,0 +1,349 @@ +//! A higher level attributes based on TokenTree, with also some shortcuts. +use std::{fmt, ops, sync::Arc}; + +use base_db::CrateId; +use cfg::CfgExpr; +use either::Either; +use intern::Interned; +use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct}; +use smallvec::{smallvec, SmallVec}; +use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode}; + +use crate::{ + db::AstDatabase, + hygiene::Hygiene, + mod_path::{ModPath, PathKind}, + name::AsName, + tt::{self, Subtree}, + InFile, +}; + +/// Syntactical attributes, without filtering of `cfg_attr`s. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct RawAttrs { + entries: Option<Arc<[Attr]>>, +} + +impl ops::Deref for RawAttrs { + type Target = [Attr]; + + fn deref(&self) -> &[Attr] { + match &self.entries { + Some(it) => &*it, + None => &[], + } + } +} + +impl RawAttrs { + pub const EMPTY: Self = Self { entries: None }; + + pub fn new(db: &dyn AstDatabase, owner: &dyn ast::HasAttrs, hygiene: &Hygiene) -> Self { + let entries = collect_attrs(owner) + .filter_map(|(id, attr)| match attr { + Either::Left(attr) => { + attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id)) + } + Either::Right(comment) => comment.doc_comment().map(|doc| Attr { + id, + input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))), + path: Interned::new(ModPath::from(crate::name!(doc))), + }), + }) + .collect::<Arc<_>>(); + + Self { entries: if entries.is_empty() { None } else { Some(entries) } } + } + + pub fn from_attrs_owner(db: &dyn AstDatabase, owner: InFile<&dyn ast::HasAttrs>) -> Self { + let hygiene = Hygiene::new(db, owner.file_id); + Self::new(db, owner.value, &hygiene) + } + + pub fn merge(&self, other: Self) -> Self { + match (&self.entries, other.entries) { + (None, None) => Self::EMPTY, + (None, entries @ Some(_)) => Self { entries }, + (Some(entries), None) => Self { entries: Some(entries.clone()) }, + (Some(a), Some(b)) => { + let last_ast_index = a.last().map_or(0, |it| it.id.ast_index() + 1) as u32; + Self { + entries: Some( + a.iter() + .cloned() + .chain(b.iter().map(|it| { + let mut it = it.clone(); + it.id.id = it.id.ast_index() as u32 + last_ast_index + | (it.id.cfg_attr_index().unwrap_or(0) as u32) + << AttrId::AST_INDEX_BITS; + it + })) + .collect(), + ), + } + } + } + } + + /// Processes `cfg_attr`s, returning the resulting semantic `Attrs`. + // FIXME: This should return a different type + pub fn filter(self, db: &dyn AstDatabase, krate: CrateId) -> RawAttrs { + let has_cfg_attrs = self + .iter() + .any(|attr| attr.path.as_ident().map_or(false, |name| *name == crate::name![cfg_attr])); + if !has_cfg_attrs { + return self; + } + + let crate_graph = db.crate_graph(); + let new_attrs = self + .iter() + .flat_map(|attr| -> SmallVec<[_; 1]> { + let is_cfg_attr = + attr.path.as_ident().map_or(false, |name| *name == crate::name![cfg_attr]); + if !is_cfg_attr { + return smallvec![attr.clone()]; + } + + let subtree = match attr.token_tree_value() { + Some(it) => it, + _ => return smallvec![attr.clone()], + }; + + let (cfg, parts) = match parse_cfg_attr_input(subtree) { + Some(it) => it, + None => return smallvec![attr.clone()], + }; + let index = attr.id; + let attrs = + parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| { + let tree = Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: attr.to_vec(), + }; + // FIXME hygiene + let hygiene = Hygiene::new_unhygienic(); + Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx)) + }); + + let cfg_options = &crate_graph[krate].cfg_options; + let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() }; + let cfg = CfgExpr::parse(&cfg); + if cfg_options.check(&cfg) == Some(false) { + smallvec![] + } else { + cov_mark::hit!(cfg_attr_active); + + attrs.collect() + } + }) + .collect(); + + RawAttrs { entries: Some(new_attrs) } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct AttrId { + id: u32, +} + +// FIXME: This only handles a single level of cfg_attr nesting +// that is `#[cfg_attr(all(), cfg_attr(all(), cfg(any())))]` breaks again +impl AttrId { + const CFG_ATTR_BITS: usize = 7; + const AST_INDEX_MASK: usize = 0x00FF_FFFF; + const AST_INDEX_BITS: usize = Self::AST_INDEX_MASK.count_ones() as usize; + const CFG_ATTR_SET_BITS: u32 = 1 << 31; + + pub fn ast_index(&self) -> usize { + self.id as usize & Self::AST_INDEX_MASK + } + + pub fn cfg_attr_index(&self) -> Option<usize> { + if self.id & Self::CFG_ATTR_SET_BITS == 0 { + None + } else { + Some(self.id as usize >> Self::AST_INDEX_BITS) + } + } + + pub fn with_cfg_attr(self, idx: usize) -> AttrId { + AttrId { id: self.id | (idx as u32) << Self::AST_INDEX_BITS | Self::CFG_ATTR_SET_BITS } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Attr { + pub id: AttrId, + pub path: Interned<ModPath>, + pub input: Option<Interned<AttrInput>>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum AttrInput { + /// `#[attr = "string"]` + Literal(SmolStr), + /// `#[attr(subtree)]` + TokenTree(tt::Subtree, mbe::TokenMap), +} + +impl fmt::Display for AttrInput { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + AttrInput::Literal(lit) => write!(f, " = \"{}\"", lit.escape_debug()), + AttrInput::TokenTree(subtree, _) => subtree.fmt(f), + } + } +} + +impl Attr { + fn from_src( + db: &dyn AstDatabase, + ast: ast::Meta, + hygiene: &Hygiene, + id: AttrId, + ) -> Option<Attr> { + let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?); + let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() { + let value = match lit.kind() { + ast::LiteralKind::String(string) => string.value()?.into(), + _ => lit.syntax().first_token()?.text().trim_matches('"').into(), + }; + Some(Interned::new(AttrInput::Literal(value))) + } else if let Some(tt) = ast.token_tree() { + let (tree, map) = syntax_node_to_token_tree(tt.syntax()); + Some(Interned::new(AttrInput::TokenTree(tree, map))) + } else { + None + }; + Some(Attr { id, path, input }) + } + + fn from_tt( + db: &dyn AstDatabase, + tt: &tt::Subtree, + hygiene: &Hygiene, + id: AttrId, + ) -> Option<Attr> { + let (parse, _) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); + let ast = ast::Meta::cast(parse.syntax_node())?; + + Self::from_src(db, ast, hygiene, id) + } + + pub fn path(&self) -> &ModPath { + &self.path + } +} + +impl Attr { + /// #[path = "string"] + pub fn string_value(&self) -> Option<&SmolStr> { + match self.input.as_deref()? { + AttrInput::Literal(it) => Some(it), + _ => None, + } + } + + /// #[path(ident)] + pub fn single_ident_value(&self) -> Option<&tt::Ident> { + match self.input.as_deref()? { + AttrInput::TokenTree(subtree, _) => match &*subtree.token_trees { + [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident), + _ => None, + }, + _ => None, + } + } + + /// #[path TokenTree] + pub fn token_tree_value(&self) -> Option<&Subtree> { + match self.input.as_deref()? { + AttrInput::TokenTree(subtree, _) => Some(subtree), + _ => None, + } + } + + /// Parses this attribute as a token tree consisting of comma separated paths. + pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> { + let args = self.token_tree_value()?; + + if args.delimiter.kind != DelimiterKind::Parenthesis { + return None; + } + let paths = args + .token_trees + .split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(Punct { char: ',', .. })))) + .filter_map(|tts| { + if tts.is_empty() { + return None; + } + let segments = tts.iter().filter_map(|tt| match tt { + tt::TokenTree::Leaf(tt::Leaf::Ident(id)) => Some(id.as_name()), + _ => None, + }); + Some(ModPath::from_segments(PathKind::Plain, segments)) + }); + + Some(paths) + } +} + +pub fn collect_attrs( + owner: &dyn ast::HasAttrs, +) -> impl Iterator<Item = (AttrId, Either<ast::Attr, ast::Comment>)> { + let inner_attrs = inner_attributes(owner.syntax()).into_iter().flatten(); + let outer_attrs = + ast::AttrDocCommentIter::from_syntax_node(owner.syntax()).filter(|el| match el { + Either::Left(attr) => attr.kind().is_outer(), + Either::Right(comment) => comment.is_outer(), + }); + outer_attrs.chain(inner_attrs).enumerate().map(|(id, attr)| (AttrId { id: id as u32 }, attr)) +} + +fn inner_attributes( + syntax: &SyntaxNode, +) -> Option<impl Iterator<Item = Either<ast::Attr, ast::Comment>>> { + let node = match_ast! { + match syntax { + ast::SourceFile(_) => syntax.clone(), + ast::ExternBlock(it) => it.extern_item_list()?.syntax().clone(), + ast::Fn(it) => it.body()?.stmt_list()?.syntax().clone(), + ast::Impl(it) => it.assoc_item_list()?.syntax().clone(), + ast::Module(it) => it.item_list()?.syntax().clone(), + ast::BlockExpr(it) => { + use syntax::SyntaxKind::{BLOCK_EXPR , EXPR_STMT}; + // Block expressions accept outer and inner attributes, but only when they are the outer + // expression of an expression statement or the final expression of another block expression. + let may_carry_attributes = matches!( + it.syntax().parent().map(|it| it.kind()), + Some(BLOCK_EXPR | EXPR_STMT) + ); + if !may_carry_attributes { + return None + } + syntax.clone() + }, + _ => return None, + } + }; + + let attrs = ast::AttrDocCommentIter::from_syntax_node(&node).filter(|el| match el { + Either::Left(attr) => attr.kind().is_inner(), + Either::Right(comment) => comment.is_inner(), + }); + Some(attrs) +} + +// Input subtree is: `(cfg, $(attr),+)` +// Split it up into a `cfg` subtree and the `attr` subtrees. +pub fn parse_cfg_attr_input( + subtree: &Subtree, +) -> Option<(&[tt::TokenTree], impl Iterator<Item = &[tt::TokenTree]>)> { + let mut parts = subtree + .token_trees + .split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(Punct { char: ',', .. })))); + let cfg = parts.next()?; + Some((cfg, parts.filter(|it| !it.is_empty()))) +} diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs index 58d192f9f..906ca991d 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs @@ -1,6 +1,6 @@ //! Builtin attributes. -use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind}; +use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind}; macro_rules! register_builtin { ( $(($name:ident, $variant:ident) => $expand:ident),* ) => { @@ -97,7 +97,7 @@ fn derive_attr_expand( let loc = db.lookup_intern_macro_call(id); let derives = match &loc.kind { MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0, - _ => return ExpandResult::ok(Default::default()), + _ => return ExpandResult::ok(tt::Subtree::empty()), }; pseudo_derive_attr_expansion(tt, derives) } @@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion( tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char, spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) }; diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs index 8966047c9..060a68054 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs @@ -3,11 +3,11 @@ use base_db::{CrateOrigin, LangCrateOrigin}; use tracing::debug; +use crate::tt::{self, TokenId}; use syntax::{ ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName}, match_ast, }; -use tt::TokenId; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId}; @@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> { })?; let name_token_id = token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified); - let name_token = tt::Ident { id: name_token_id, text: name.text().into() }; + let name_token = tt::Ident { span: name_token_id, text: name.text().into() }; let param_types = params .into_iter() .flat_map(|param_list| param_list.type_or_const_params()) @@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> { let ty = param .ty() .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0) - .unwrap_or_default(); + .unwrap_or_else(tt::Subtree::empty); Some(ty) } else { None @@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> { fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> { let info = match parse_adt(tt) { Ok(info) => info, - Err(e) => return ExpandResult::only_err(e), + Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e), }; let (params, args): (Vec<_>, Vec<_>) = info .param_types @@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu .enumerate() .map(|(idx, param_ty)| { let ident = tt::Leaf::Ident(tt::Ident { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), text: format!("T{idx}").into(), }); let ident_ = ident.clone(); diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs index 5522bdf3b..9f3fa73d4 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs @@ -9,7 +9,9 @@ use syntax::{ SmolStr, }; -use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; +use crate::{ + db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc, +}; macro_rules! register_builtin { ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => { @@ -61,7 +63,7 @@ macro_rules! register_builtin { }; } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct ExpandedEager { pub(crate) subtree: tt::Subtree, /// The included file ID of the include macro. @@ -116,7 +118,7 @@ register_builtin! { } const DOLLAR_CRATE: tt::Ident = - tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() }; + tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() }; fn module_path_expand( _db: &dyn AstDatabase, @@ -162,7 +164,7 @@ fn stringify_expand( _id: MacroCallId, tt: &tt::Subtree, ) -> ExpandResult<tt::Subtree> { - let pretty = tt::pretty(&tt.token_trees); + let pretty = ::tt::pretty(&tt.token_trees); let expanded = quote! { #pretty @@ -194,11 +196,11 @@ fn assert_expand( let expanded = match &*args { [cond, panic_args @ ..] => { let comma = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }; let cond = cond.clone(); @@ -247,7 +249,10 @@ fn format_args_expand( let mut args = parse_exprs_with_sep(tt, ','); if args.is_empty() { - return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into()); + return ExpandResult::with_err( + tt::Subtree::empty(), + mbe::ExpandError::NoMatchingRule.into(), + ); } for arg in &mut args { // Remove `key =`. @@ -282,7 +287,7 @@ fn asm_expand( for tt in tt.token_trees.chunks(2) { match tt { [tt::TokenTree::Leaf(tt::Leaf::Literal(lit))] - | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] => + | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] => { let krate = DOLLAR_CRATE.clone(); literals.push(quote!(#krate::format_args!(#lit);)); @@ -400,7 +405,7 @@ fn concat_expand( // FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses // to ensure the right parsing order, so skip the parentheses here. Ideally we'd // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623 - if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t { + if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t { if let [tt] = &**token_trees { if delim.kind == tt::DelimiterKind::Parenthesis { t = tt; @@ -459,9 +464,7 @@ fn concat_bytes_expand( } } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (), - tt::TokenTree::Subtree(tree) - if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) => - { + tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => { if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) { err.get_or_insert(e); break; @@ -473,7 +476,7 @@ fn concat_bytes_expand( } } } - let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() }; ExpandResult { value: ExpandedEager::new(quote!([#ident])), err } } @@ -521,7 +524,7 @@ fn concat_idents_expand( } } } - let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() }; ExpandResult { value: ExpandedEager::new(quote!(#ident)), err } } @@ -572,7 +575,10 @@ fn include_expand( Ok((subtree, file_id)) => { ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) }) } - Err(e) => ExpandResult::only_err(e), + Err(e) => ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ), } } @@ -582,15 +588,18 @@ fn include_bytes_expand( tt: &tt::Subtree, ) -> ExpandResult<ExpandedEager> { if let Err(e) = parse_string(tt) { - return ExpandResult::only_err(e); + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ); } // FIXME: actually read the file here if the user asked for macro expansion let res = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: r#"b"""#.into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }; ExpandResult::ok(ExpandedEager::new(res)) @@ -603,7 +612,12 @@ fn include_str_expand( ) -> ExpandResult<ExpandedEager> { let path = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; // FIXME: we're not able to read excluded files (which is most of them because @@ -635,7 +649,12 @@ fn env_expand( ) -> ExpandResult<ExpandedEager> { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; let mut err = None; @@ -666,7 +685,12 @@ fn option_env_expand( ) -> ExpandResult<ExpandedEager> { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; let expanded = match get_env_inner(db, arg_id, &key) { diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs index b28e60187..76016274f 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs @@ -14,7 +14,7 @@ use syntax::{ use crate::{ ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup, - hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, + hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, }; @@ -25,7 +25,7 @@ use crate::{ /// an error will be emitted. /// /// Actual max for `analysis-stats .` at some point: 30672. -static TOKEN_LIMIT: Limit = Limit::new(524_288); +static TOKEN_LIMIT: Limit = Limit::new(1_048_576); #[derive(Debug, Clone, Eq, PartialEq)] pub enum TokenExpander { @@ -168,12 +168,14 @@ pub fn expand_speculative( // Attributes may have an input token tree, build the subtree and map for this as well // then try finding a token id for our token if it is inside this input subtree. let item = ast::Item::cast(speculative_args.clone())?; - item.doc_comments_and_attrs().nth(invoc_attr_index as usize).and_then(Either::left) + item.doc_comments_and_attrs() + .nth(invoc_attr_index.ast_index()) + .and_then(Either::left) }?; match attr.token_tree() { Some(token_tree) => { let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax()); - tree.delimiter = None; + tree.delimiter = tt::Delimiter::unspecified(); let shift = mbe::Shift::new(&tt); shift.shift_all(&mut tree); @@ -208,7 +210,7 @@ pub fn expand_speculative( // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { - tt.delimiter = None; + tt.delimiter = tt::Delimiter::unspecified(); expander.expand(db, loc.krate, &tt, attr_arg.as_ref()) } MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { @@ -314,13 +316,13 @@ fn macro_arg( if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.delimiter = None; + tt.delimiter = tt::Delimiter::unspecified(); } - Some(Arc::new((tt, tmap, fixups.undo_info))) } fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<SyntaxNode> { + // FIXME: handle `cfg_attr` (|| { let censor = match loc.kind { MacroCallKind::FnLike { .. } => return None, @@ -328,7 +330,7 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<Sy cov_mark::hit!(derive_censoring); ast::Item::cast(node.clone())? .attrs() - .take(derive_attr_index as usize + 1) + .take(derive_attr_index.ast_index() + 1) // FIXME, this resolution should not be done syntactically // derive is a proper macro now, no longer builtin // But we do not have resolution at this stage, this means @@ -343,7 +345,7 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<Sy cov_mark::hit!(attribute_macro_attr_censoring); ast::Item::cast(node.clone())? .doc_comments_and_attrs() - .nth(invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .and_then(Either::left) .map(|attr| attr.syntax().clone()) .into_iter() @@ -476,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult<tt:: let macro_arg = match db.macro_arg(id) { Some(it) => it, None => { - return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into())) + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("No arguments for proc-macro".into()), + ) } }; diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs index a1474c44e..dfab7ec92 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs @@ -108,7 +108,7 @@ pub fn expand_eager_macro( .value .token_tree() .map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0) - .unwrap_or_default(); + .unwrap_or_else(tt::Subtree::empty); let ast_map = db.ast_id_map(macro_call.file_id); let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(¯o_call.value)); @@ -165,9 +165,9 @@ pub fn expand_eager_macro( } } -fn to_subtree(node: &SyntaxNode) -> tt::Subtree { +fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree { let mut subtree = mbe::syntax_node_to_token_tree(node).0; - subtree.delimiter = None; + subtree.delimiter = crate::tt::Delimiter::unspecified(); subtree } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs index 75d364d5f..c811d1c66 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs @@ -9,7 +9,7 @@ use syntax::{ ast::{self, AstNode, HasLoopBody}, match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, }; -use tt::Subtree; +use tt::token_id::Subtree; /// The result of calculating fixes for a syntax node -- a bunch of changes /// (appending to and replacing nodes), the information that is needed to @@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups( tt.token_trees = tts .into_iter() .filter(|tt| match tt { - tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID), + tt::TokenTree::Leaf(leaf) => { + token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID) + } tt::TokenTree::Subtree(st) => { - st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID)) + token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID) } }) .flat_map(|tt| match tt { @@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups( SmallVec::from_const([tt.into()]) } tt::TokenTree::Leaf(leaf) => { - if let Some(id) = token_map.synthetic_token_id(leaf.id()) { + if let Some(id) = token_map.synthetic_token_id(*leaf.span()) { let original = undo_info.original[id.0 as usize].clone(); - if original.delimiter.is_none() { + if original.delimiter.kind == tt::DelimiterKind::Invisible { original.token_trees.into() } else { SmallVec::from_const([original.into()]) @@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups( mod tests { use expect_test::{expect, Expect}; + use crate::tt; + use super::reverse_fixups; // The following three functions are only meant to check partial structural equivalence of @@ -341,7 +345,7 @@ mod tests { } fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool { - a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind) + a.delimiter.kind == b.delimiter.kind && a.token_trees.len() == b.token_trees.len() && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b)) } @@ -386,7 +390,7 @@ mod tests { let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node()); assert!( check_subtree_eq(&tt, &original_as_tt), - "different token tree: {tt:?}, {original_as_tt:?}" + "different token tree: {tt:?},\n{original_as_tt:?}" ); } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs b/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs index df1e20256..2300ee9d0 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs @@ -128,7 +128,7 @@ struct HygieneInfo { attr_input_or_mac_def_start: Option<InFile<TextSize>>, macro_def: Arc<TokenExpander>, - macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, + macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, macro_arg_shift: mbe::Shift, exp_map: Arc<mbe::TokenMap>, } @@ -191,7 +191,7 @@ fn make_hygiene_info( let tt = ast_id .to_node(db) .doc_comments_and_attrs() - .nth(invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .and_then(Either::left)? .token_tree()?; Some(InFile::new(ast_id.file_id, tt)) diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs b/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs index bc5f9f3b8..a52716cc0 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs @@ -17,10 +17,13 @@ pub mod proc_macro; pub mod quote; pub mod eager; pub mod mod_path; +pub mod attrs; mod fixup; pub use mbe::{Origin, ValueResult}; +use ::tt::token_id as tt; + use std::{fmt, hash::Hash, iter, sync::Arc}; use base_db::{ @@ -37,6 +40,7 @@ use syntax::{ use crate::{ ast_id_map::FileAstId, + attrs::AttrId, builtin_attr_macro::BuiltinAttrExpander, builtin_derive_macro::BuiltinDeriveExpander, builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander}, @@ -51,6 +55,7 @@ pub type ExpandResult<T> = ValueResult<T, ExpandError>; pub enum ExpandError { UnresolvedProcMacro(CrateId), Mbe(mbe::ExpandError), + RecursionOverflowPosioned, Other(Box<str>), } @@ -65,6 +70,9 @@ impl fmt::Display for ExpandError { match self { ExpandError::UnresolvedProcMacro(_) => f.write_str("unresolved proc-macro"), ExpandError::Mbe(it) => it.fmt(f), + ExpandError::RecursionOverflowPosioned => { + f.write_str("overflow expanding the original macro") + } ExpandError::Other(it) => f.write_str(it), } } @@ -114,6 +122,7 @@ pub struct MacroDefId { pub krate: CrateId, pub kind: MacroDefKind, pub local_inner: bool, + pub allow_internal_unsafe: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -145,7 +154,7 @@ pub enum MacroCallKind { /// /// Outer attributes are counted first, then inner attributes. This does not support /// out-of-line modules, which may have attributes spread across 2 files! - derive_attr_index: u32, + derive_attr_index: AttrId, /// Index of the derive macro in the derive attribute derive_index: u32, }, @@ -156,7 +165,7 @@ pub enum MacroCallKind { /// /// Outer attributes are counted first, then inner attributes. This does not support /// out-of-line modules, which may have attributes spread across 2 files! - invoc_attr_index: u32, + invoc_attr_index: AttrId, /// Whether this attribute is the `#[derive]` attribute. is_derive: bool, }, @@ -261,10 +270,11 @@ impl HirFileId { }); let attr_input_or_mac_def = def.or_else(|| match loc.kind { MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => { + // FIXME: handle `cfg_attr` let tt = ast_id .to_node(db) .doc_comments_and_attrs() - .nth(invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .and_then(Either::left)? .token_tree()?; Some(InFile::new(ast_id.file_id, tt)) @@ -353,6 +363,14 @@ impl HirFileId { } } + #[inline] + pub fn file_id(self) -> Option<FileId> { + match self.0 & Self::MACRO_FILE_TAG_MASK { + 0 => Some(FileId(self.0)), + _ => None, + } + } + fn repr(self) -> HirFileIdRepr { match self.0 & Self::MACRO_FILE_TAG_MASK { 0 => HirFileIdRepr::FileId(FileId(self.0)), @@ -397,8 +415,7 @@ impl MacroDefId { } } -// FIXME: attribute indices do not account for `cfg_attr`, which means that we'll strip the whole -// `cfg_attr` instead of just one of the attributes it expands to +// FIXME: attribute indices do not account for nested `cfg_attr` impl MacroCallKind { /// Returns the file containing the macro invocation. @@ -419,7 +436,7 @@ impl MacroCallKind { // FIXME: handle `cfg_attr` ast_id.with_value(ast_id.to_node(db)).map(|it| { it.doc_comments_and_attrs() - .nth(*derive_attr_index as usize) + .nth(derive_attr_index.ast_index()) .and_then(|it| match it { Either::Left(attr) => Some(attr.syntax().clone()), Either::Right(_) => None, @@ -431,7 +448,7 @@ impl MacroCallKind { // FIXME: handle `cfg_attr` ast_id.with_value(ast_id.to_node(db)).map(|it| { it.doc_comments_and_attrs() - .nth(*invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .and_then(|it| match it { Either::Left(attr) => Some(attr.syntax().clone()), Either::Right(_) => None, @@ -488,19 +505,21 @@ impl MacroCallKind { MacroCallKind::FnLike { ast_id, .. } => ast_id.to_node(db).syntax().text_range(), MacroCallKind::Derive { ast_id, derive_attr_index, .. } => { // FIXME: should be the range of the macro name, not the whole derive + // FIXME: handle `cfg_attr` ast_id .to_node(db) .doc_comments_and_attrs() - .nth(derive_attr_index as usize) + .nth(derive_attr_index.ast_index()) .expect("missing derive") .expect_left("derive is a doc comment?") .syntax() .text_range() } + // FIXME: handle `cfg_attr` MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => ast_id .to_node(db) .doc_comments_and_attrs() - .nth(invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .expect("missing attribute") .expect_left("attribute macro is a doc comment?") .syntax() @@ -592,9 +611,10 @@ impl ExpansionInfo { let token_range = token.value.text_range(); match &loc.kind { MacroCallKind::Attr { attr_args, invoc_attr_index, is_derive, .. } => { + // FIXME: handle `cfg_attr` let attr = item .doc_comments_and_attrs() - .nth(*invoc_attr_index as usize) + .nth(invoc_attr_index.ast_index()) .and_then(Either::left)?; match attr.token_tree() { Some(token_tree) @@ -1031,3 +1051,5 @@ impl ExpandTo { pub struct UnresolvedMacro { pub path: ModPath, } + +intern::impl_internable!(ModPath, attrs::AttrInput); diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs index e8b3e312a..c3462beac 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs @@ -2,7 +2,7 @@ use std::fmt; -use syntax::{ast, SmolStr, SyntaxKind}; +use syntax::{ast, utils::is_raw_identifier, SmolStr}; /// `Name` is a wrapper around string, which is used in hir for both references /// and declarations. In theory, names should also carry hygiene info, but we are @@ -33,11 +33,6 @@ impl fmt::Display for Name { } } -fn is_raw_identifier(name: &str) -> bool { - let is_keyword = SyntaxKind::from_keyword(name).is_some(); - is_keyword && !matches!(name, "self" | "crate" | "super" | "Self") -} - impl<'a> fmt::Display for UnescapedName<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.0 .0 { @@ -133,6 +128,14 @@ impl Name { } } + /// Returns the text this name represents if it isn't a tuple field. + pub fn as_str(&self) -> Option<&str> { + match &self.0 { + Repr::Text(it) => Some(it), + _ => None, + } + } + /// Returns the textual representation of this name as a [`SmolStr`]. /// Prefer using this over [`ToString::to_string`] if possible as this conversion is cheaper in /// the general case. @@ -183,7 +186,7 @@ impl AsName for ast::NameOrNameRef { } } -impl AsName for tt::Ident { +impl<Span> AsName for tt::Ident<Span> { fn as_name(&self) -> Name { Name::resolve(&self.text) } @@ -339,6 +342,7 @@ pub mod known { recursion_limit, feature, // known methods of lang items + call_once, eq, ne, ge, diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs index 5afdcc0e6..3f4d2540c 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs @@ -3,7 +3,7 @@ use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind}; use stdx::never; -use crate::{db::AstDatabase, ExpandError, ExpandResult}; +use crate::{db::AstDatabase, tt, ExpandError, ExpandResult}; #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] pub struct ProcMacroExpander { @@ -39,7 +39,10 @@ impl ProcMacroExpander { Ok(proc_macros) => proc_macros, Err(_) => { never!("Non-dummy expander even though there are no proc macros"); - return ExpandResult::only_err(ExpandError::Other("Internal error".into())); + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("Internal error".into()), + ); } }; let proc_macro = match proc_macros.get(id.0 as usize) { @@ -50,7 +53,10 @@ impl ProcMacroExpander { proc_macros.len(), id.0 ); - return ExpandResult::only_err(ExpandError::Other("Internal error".into())); + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("Internal error".into()), + ); } }; @@ -69,13 +75,17 @@ impl ProcMacroExpander { } } ProcMacroExpansionError::System(text) - | ProcMacroExpansionError::Panic(text) => { - ExpandResult::only_err(ExpandError::Other(text.into())) - } + | ProcMacroExpansionError::Panic(text) => ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other(text.into()), + ), }, } } - None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)), + None => ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::UnresolvedProcMacro(self.krate), + ), } } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs index c0a7bc7ca..63586f9da 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs @@ -9,17 +9,18 @@ #[macro_export] macro_rules! __quote { () => { - Vec::<tt::TokenTree>::new() + Vec::<crate::tt::TokenTree>::new() }; ( @SUBTREE $delim:ident $($tt:tt)* ) => { { let children = $crate::__quote!($($tt)*); - tt::Subtree { - delimiter: Some(tt::Delimiter { - kind: tt::DelimiterKind::$delim, - id: tt::TokenId::unspecified(), - }), + crate::tt::Subtree { + delimiter: crate::tt::Delimiter { + kind: crate::tt::DelimiterKind::$delim, + open: crate::tt::TokenId::unspecified(), + close: crate::tt::TokenId::unspecified(), + }, token_trees: $crate::quote::IntoTt::to_tokens(children), } } @@ -28,10 +29,10 @@ macro_rules! __quote { ( @PUNCT $first:literal ) => { { vec![ - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Alone, + span: crate::tt::TokenId::unspecified(), }).into() ] } @@ -40,15 +41,15 @@ macro_rules! __quote { ( @PUNCT $first:literal, $sec:literal ) => { { vec![ - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, - spacing: tt::Spacing::Joint, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Joint, + span: crate::tt::TokenId::unspecified(), }).into(), - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $sec, - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Alone, + span: crate::tt::TokenId::unspecified(), }).into() ] } @@ -67,7 +68,7 @@ macro_rules! __quote { ( ## $first:ident $($tail:tt)* ) => { { - let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<tt::TokenTree>>(); + let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<crate::tt::TokenTree>>(); let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); tokens.append(&mut tail_tokens); tokens @@ -86,9 +87,9 @@ macro_rules! __quote { // Ident ( $tt:ident ) => { vec![ { - tt::Leaf::Ident(tt::Ident { + crate::tt::Leaf::Ident(crate::tt::Ident { text: stringify!($tt).into(), - id: tt::TokenId::unspecified(), + span: crate::tt::TokenId::unspecified(), }).into() }] }; @@ -127,42 +128,42 @@ macro_rules! quote { } pub(crate) trait IntoTt { - fn to_subtree(self) -> tt::Subtree; - fn to_tokens(self) -> Vec<tt::TokenTree>; + fn to_subtree(self) -> crate::tt::Subtree; + fn to_tokens(self) -> Vec<crate::tt::TokenTree>; } -impl IntoTt for Vec<tt::TokenTree> { - fn to_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: None, token_trees: self } +impl IntoTt for Vec<crate::tt::TokenTree> { + fn to_subtree(self) -> crate::tt::Subtree { + crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self } } - fn to_tokens(self) -> Vec<tt::TokenTree> { + fn to_tokens(self) -> Vec<crate::tt::TokenTree> { self } } -impl IntoTt for tt::Subtree { - fn to_subtree(self) -> tt::Subtree { +impl IntoTt for crate::tt::Subtree { + fn to_subtree(self) -> crate::tt::Subtree { self } - fn to_tokens(self) -> Vec<tt::TokenTree> { - vec![tt::TokenTree::Subtree(self)] + fn to_tokens(self) -> Vec<crate::tt::TokenTree> { + vec![crate::tt::TokenTree::Subtree(self)] } } pub(crate) trait ToTokenTree { - fn to_token(self) -> tt::TokenTree; + fn to_token(self) -> crate::tt::TokenTree; } -impl ToTokenTree for tt::TokenTree { - fn to_token(self) -> tt::TokenTree { +impl ToTokenTree for crate::tt::TokenTree { + fn to_token(self) -> crate::tt::TokenTree { self } } -impl ToTokenTree for tt::Subtree { - fn to_token(self) -> tt::TokenTree { +impl ToTokenTree for crate::tt::Subtree { + fn to_token(self) -> crate::tt::TokenTree { self.into() } } @@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees { ($($ty:ty => $this:ident $im:block);*) => { $( impl ToTokenTree for $ty { - fn to_token($this) -> tt::TokenTree { - let leaf: tt::Leaf = $im.into(); + fn to_token($this) -> crate::tt::TokenTree { + let leaf: crate::tt::Leaf = $im.into(); leaf.into() } } impl ToTokenTree for &$ty { - fn to_token($this) -> tt::TokenTree { - let leaf: tt::Leaf = $im.clone().into(); + fn to_token($this) -> crate::tt::TokenTree { + let leaf: crate::tt::Leaf = $im.clone().into(); leaf.into() } } @@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - tt::Leaf => self { self }; - tt::Literal => self { self }; - tt::Ident => self { self }; - tt::Punct => self { self }; - &str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}; - String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}} + u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + crate::tt::Leaf => self { self }; + crate::tt::Literal => self { self }; + crate::tt::Ident => self { self }; + crate::tt::Punct => self { self }; + &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}; + String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}} } #[cfg(test)] @@ -223,8 +224,8 @@ mod tests { assert_eq!(quote!(#s).to_string(), "\"hello\""); } - fn mk_ident(name: &str) -> tt::Ident { - tt::Ident { text: name.into(), id: tt::TokenId::unspecified() } + fn mk_ident(name: &str) -> crate::tt::Ident { + crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() } } #[test] @@ -234,7 +235,7 @@ mod tests { let quoted = quote!(#a); assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); - assert_eq!(t, "SUBTREE $\n IDENT hello 4294967295"); + assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295"); } #[test] @@ -263,11 +264,12 @@ mod tests { let fields = [mk_ident("name"), mk_ident("id")]; let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees); - let list = tt::Subtree { - delimiter: Some(tt::Delimiter { - kind: tt::DelimiterKind::Brace, - id: tt::TokenId::unspecified(), - }), + let list = crate::tt::Subtree { + delimiter: crate::tt::Delimiter { + kind: crate::tt::DelimiterKind::Brace, + open: crate::tt::TokenId::unspecified(), + close: crate::tt::TokenId::unspecified(), + }, token_trees: fields.collect(), }; |