Merging upstream version 1.69.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

13 files changed, 566 insertions, 142 deletions

diff --git a/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml b/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml
index 77eb1fd45..5c684be03 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml
+++ b/src/tools/rust-analyzer/crates/hir-expand/Cargo.toml
@@ -2,9 +2,11 @@
 name = "hir-expand"
 version = "0.0.0"
 description = "TBD"
-license = "MIT OR Apache-2.0"
-edition = "2021"
-rust-version = "1.65"
+
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
 
 [lib]
 doctest = false
@@ -19,16 +21,18 @@ itertools = "0.10.5"
 hashbrown = { version = "0.12.1", features = [
     "inline-more",
 ], default-features = false }
-smallvec = { version = "1.10.0", features = ["const_new"] }
+smallvec.workspace = true
 
-stdx = { path = "../stdx", version = "0.0.0" }
-base-db = { path = "../base-db", version = "0.0.0" }
-cfg = { path = "../cfg", version = "0.0.0" }
-syntax = { path = "../syntax", version = "0.0.0" }
-profile = { path = "../profile", version = "0.0.0" }
-tt = { path = "../tt", version = "0.0.0" }
-mbe = { path = "../mbe", version = "0.0.0" }
-limit = { path = "../limit", version = "0.0.0" }
+# local deps
+stdx.workspace = true
+intern.workspace = true
+base-db.workspace = true
+cfg.workspace = true
+syntax.workspace = true
+profile.workspace = true
+tt.workspace = true
+mbe.workspace = true
+limit.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.0"
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs
new file mode 100644
index 000000000..5c04f8e8b
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs
@@ -0,0 +1,349 @@
+//! A higher level attributes based on TokenTree, with also some shortcuts.
+use std::{fmt, ops, sync::Arc};
+
+use base_db::CrateId;
+use cfg::CfgExpr;
+use either::Either;
+use intern::Interned;
+use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
+use smallvec::{smallvec, SmallVec};
+use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
+
+use crate::{
+    db::AstDatabase,
+    hygiene::Hygiene,
+    mod_path::{ModPath, PathKind},
+    name::AsName,
+    tt::{self, Subtree},
+    InFile,
+};
+
+/// Syntactical attributes, without filtering of `cfg_attr`s.
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
+pub struct RawAttrs {
+    entries: Option<Arc<[Attr]>>,
+}
+
+impl ops::Deref for RawAttrs {
+    type Target = [Attr];
+
+    fn deref(&self) -> &[Attr] {
+        match &self.entries {
+            Some(it) => &*it,
+            None => &[],
+        }
+    }
+}
+
+impl RawAttrs {
+    pub const EMPTY: Self = Self { entries: None };
+
+    pub fn new(db: &dyn AstDatabase, owner: &dyn ast::HasAttrs, hygiene: &Hygiene) -> Self {
+        let entries = collect_attrs(owner)
+            .filter_map(|(id, attr)| match attr {
+                Either::Left(attr) => {
+                    attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id))
+                }
+                Either::Right(comment) => comment.doc_comment().map(|doc| Attr {
+                    id,
+                    input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))),
+                    path: Interned::new(ModPath::from(crate::name!(doc))),
+                }),
+            })
+            .collect::<Arc<_>>();
+
+        Self { entries: if entries.is_empty() { None } else { Some(entries) } }
+    }
+
+    pub fn from_attrs_owner(db: &dyn AstDatabase, owner: InFile<&dyn ast::HasAttrs>) -> Self {
+        let hygiene = Hygiene::new(db, owner.file_id);
+        Self::new(db, owner.value, &hygiene)
+    }
+
+    pub fn merge(&self, other: Self) -> Self {
+        match (&self.entries, other.entries) {
+            (None, None) => Self::EMPTY,
+            (None, entries @ Some(_)) => Self { entries },
+            (Some(entries), None) => Self { entries: Some(entries.clone()) },
+            (Some(a), Some(b)) => {
+                let last_ast_index = a.last().map_or(0, |it| it.id.ast_index() + 1) as u32;
+                Self {
+                    entries: Some(
+                        a.iter()
+                            .cloned()
+                            .chain(b.iter().map(|it| {
+                                let mut it = it.clone();
+                                it.id.id = it.id.ast_index() as u32 + last_ast_index
+                                    | (it.id.cfg_attr_index().unwrap_or(0) as u32)
+                                        << AttrId::AST_INDEX_BITS;
+                                it
+                            }))
+                            .collect(),
+                    ),
+                }
+            }
+        }
+    }
+
+    /// Processes `cfg_attr`s, returning the resulting semantic `Attrs`.
+    // FIXME: This should return a different type
+    pub fn filter(self, db: &dyn AstDatabase, krate: CrateId) -> RawAttrs {
+        let has_cfg_attrs = self
+            .iter()
+            .any(|attr| attr.path.as_ident().map_or(false, |name| *name == crate::name![cfg_attr]));
+        if !has_cfg_attrs {
+            return self;
+        }
+
+        let crate_graph = db.crate_graph();
+        let new_attrs = self
+            .iter()
+            .flat_map(|attr| -> SmallVec<[_; 1]> {
+                let is_cfg_attr =
+                    attr.path.as_ident().map_or(false, |name| *name == crate::name![cfg_attr]);
+                if !is_cfg_attr {
+                    return smallvec![attr.clone()];
+                }
+
+                let subtree = match attr.token_tree_value() {
+                    Some(it) => it,
+                    _ => return smallvec![attr.clone()],
+                };
+
+                let (cfg, parts) = match parse_cfg_attr_input(subtree) {
+                    Some(it) => it,
+                    None => return smallvec![attr.clone()],
+                };
+                let index = attr.id;
+                let attrs =
+                    parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
+                        let tree = Subtree {
+                            delimiter: tt::Delimiter::unspecified(),
+                            token_trees: attr.to_vec(),
+                        };
+                        // FIXME hygiene
+                        let hygiene = Hygiene::new_unhygienic();
+                        Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
+                    });
+
+                let cfg_options = &crate_graph[krate].cfg_options;
+                let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() };
+                let cfg = CfgExpr::parse(&cfg);
+                if cfg_options.check(&cfg) == Some(false) {
+                    smallvec![]
+                } else {
+                    cov_mark::hit!(cfg_attr_active);
+
+                    attrs.collect()
+                }
+            })
+            .collect();
+
+        RawAttrs { entries: Some(new_attrs) }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AttrId {
+    id: u32,
+}
+
+// FIXME: This only handles a single level of cfg_attr nesting
+// that is `#[cfg_attr(all(), cfg_attr(all(), cfg(any())))]` breaks again
+impl AttrId {
+    const CFG_ATTR_BITS: usize = 7;
+    const AST_INDEX_MASK: usize = 0x00FF_FFFF;
+    const AST_INDEX_BITS: usize = Self::AST_INDEX_MASK.count_ones() as usize;
+    const CFG_ATTR_SET_BITS: u32 = 1 << 31;
+
+    pub fn ast_index(&self) -> usize {
+        self.id as usize & Self::AST_INDEX_MASK
+    }
+
+    pub fn cfg_attr_index(&self) -> Option<usize> {
+        if self.id & Self::CFG_ATTR_SET_BITS == 0 {
+            None
+        } else {
+            Some(self.id as usize >> Self::AST_INDEX_BITS)
+        }
+    }
+
+    pub fn with_cfg_attr(self, idx: usize) -> AttrId {
+        AttrId { id: self.id | (idx as u32) << Self::AST_INDEX_BITS | Self::CFG_ATTR_SET_BITS }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Attr {
+    pub id: AttrId,
+    pub path: Interned<ModPath>,
+    pub input: Option<Interned<AttrInput>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum AttrInput {
+    /// `#[attr = "string"]`
+    Literal(SmolStr),
+    /// `#[attr(subtree)]`
+    TokenTree(tt::Subtree, mbe::TokenMap),
+}
+
+impl fmt::Display for AttrInput {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            AttrInput::Literal(lit) => write!(f, " = \"{}\"", lit.escape_debug()),
+            AttrInput::TokenTree(subtree, _) => subtree.fmt(f),
+        }
+    }
+}
+
+impl Attr {
+    fn from_src(
+        db: &dyn AstDatabase,
+        ast: ast::Meta,
+        hygiene: &Hygiene,
+        id: AttrId,
+    ) -> Option<Attr> {
+        let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?);
+        let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() {
+            let value = match lit.kind() {
+                ast::LiteralKind::String(string) => string.value()?.into(),
+                _ => lit.syntax().first_token()?.text().trim_matches('"').into(),
+            };
+            Some(Interned::new(AttrInput::Literal(value)))
+        } else if let Some(tt) = ast.token_tree() {
+            let (tree, map) = syntax_node_to_token_tree(tt.syntax());
+            Some(Interned::new(AttrInput::TokenTree(tree, map)))
+        } else {
+            None
+        };
+        Some(Attr { id, path, input })
+    }
+
+    fn from_tt(
+        db: &dyn AstDatabase,
+        tt: &tt::Subtree,
+        hygiene: &Hygiene,
+        id: AttrId,
+    ) -> Option<Attr> {
+        let (parse, _) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem);
+        let ast = ast::Meta::cast(parse.syntax_node())?;
+
+        Self::from_src(db, ast, hygiene, id)
+    }
+
+    pub fn path(&self) -> &ModPath {
+        &self.path
+    }
+}
+
+impl Attr {
+    /// #[path = "string"]
+    pub fn string_value(&self) -> Option<&SmolStr> {
+        match self.input.as_deref()? {
+            AttrInput::Literal(it) => Some(it),
+            _ => None,
+        }
+    }
+
+    /// #[path(ident)]
+    pub fn single_ident_value(&self) -> Option<&tt::Ident> {
+        match self.input.as_deref()? {
+            AttrInput::TokenTree(subtree, _) => match &*subtree.token_trees {
+                [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident),
+                _ => None,
+            },
+            _ => None,
+        }
+    }
+
+    /// #[path TokenTree]
+    pub fn token_tree_value(&self) -> Option<&Subtree> {
+        match self.input.as_deref()? {
+            AttrInput::TokenTree(subtree, _) => Some(subtree),
+            _ => None,
+        }
+    }
+
+    /// Parses this attribute as a token tree consisting of comma separated paths.
+    pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
+        let args = self.token_tree_value()?;
+
+        if args.delimiter.kind != DelimiterKind::Parenthesis {
+            return None;
+        }
+        let paths = args
+            .token_trees
+            .split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(Punct { char: ',', .. }))))
+            .filter_map(|tts| {
+                if tts.is_empty() {
+                    return None;
+                }
+                let segments = tts.iter().filter_map(|tt| match tt {
+                    tt::TokenTree::Leaf(tt::Leaf::Ident(id)) => Some(id.as_name()),
+                    _ => None,
+                });
+                Some(ModPath::from_segments(PathKind::Plain, segments))
+            });
+
+        Some(paths)
+    }
+}
+
+pub fn collect_attrs(
+    owner: &dyn ast::HasAttrs,
+) -> impl Iterator<Item = (AttrId, Either<ast::Attr, ast::Comment>)> {
+    let inner_attrs = inner_attributes(owner.syntax()).into_iter().flatten();
+    let outer_attrs =
+        ast::AttrDocCommentIter::from_syntax_node(owner.syntax()).filter(|el| match el {
+            Either::Left(attr) => attr.kind().is_outer(),
+            Either::Right(comment) => comment.is_outer(),
+        });
+    outer_attrs.chain(inner_attrs).enumerate().map(|(id, attr)| (AttrId { id: id as u32 }, attr))
+}
+
+fn inner_attributes(
+    syntax: &SyntaxNode,
+) -> Option<impl Iterator<Item = Either<ast::Attr, ast::Comment>>> {
+    let node = match_ast! {
+        match syntax {
+            ast::SourceFile(_) => syntax.clone(),
+            ast::ExternBlock(it) => it.extern_item_list()?.syntax().clone(),
+            ast::Fn(it) => it.body()?.stmt_list()?.syntax().clone(),
+            ast::Impl(it) => it.assoc_item_list()?.syntax().clone(),
+            ast::Module(it) => it.item_list()?.syntax().clone(),
+            ast::BlockExpr(it) => {
+                use syntax::SyntaxKind::{BLOCK_EXPR , EXPR_STMT};
+                // Block expressions accept outer and inner attributes, but only when they are the outer
+                // expression of an expression statement or the final expression of another block expression.
+                let may_carry_attributes = matches!(
+                    it.syntax().parent().map(|it| it.kind()),
+                     Some(BLOCK_EXPR | EXPR_STMT)
+                );
+                if !may_carry_attributes {
+                    return None
+                }
+                syntax.clone()
+            },
+            _ => return None,
+        }
+    };
+
+    let attrs = ast::AttrDocCommentIter::from_syntax_node(&node).filter(|el| match el {
+        Either::Left(attr) => attr.kind().is_inner(),
+        Either::Right(comment) => comment.is_inner(),
+    });
+    Some(attrs)
+}
+
+// Input subtree is: `(cfg, $(attr),+)`
+// Split it up into a `cfg` subtree and the `attr` subtrees.
+pub fn parse_cfg_attr_input(
+    subtree: &Subtree,
+) -> Option<(&[tt::TokenTree], impl Iterator<Item = &[tt::TokenTree]>)> {
+    let mut parts = subtree
+        .token_trees
+        .split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(Punct { char: ',', .. }))));
+    let cfg = parts.next()?;
+    Some((cfg, parts.filter(|it| !it.is_empty())))
+}
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs
index 58d192f9f..906ca991d 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_attr_macro.rs
@@ -1,6 +1,6 @@
 //! Builtin attributes.
 
-use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind};
+use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
 
 macro_rules! register_builtin {
     ( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@@ -97,7 +97,7 @@ fn derive_attr_expand(
     let loc = db.lookup_intern_macro_call(id);
     let derives = match &loc.kind {
         MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
-        _ => return ExpandResult::ok(Default::default()),
+        _ => return ExpandResult::ok(tt::Subtree::empty()),
     };
     pseudo_derive_attr_expansion(tt, derives)
 }
@@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
         tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
             char,
             spacing: tt::Spacing::Alone,
-            id: tt::TokenId::unspecified(),
+            span: tt::TokenId::unspecified(),
         }))
     };
 
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs
index 8966047c9..060a68054 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs
@@ -3,11 +3,11 @@
 use base_db::{CrateOrigin, LangCrateOrigin};
 use tracing::debug;
 
+use crate::tt::{self, TokenId};
 use syntax::{
     ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
     match_ast,
 };
-use tt::TokenId;
 
 use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
 
@@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
     })?;
     let name_token_id =
         token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
-    let name_token = tt::Ident { id: name_token_id, text: name.text().into() };
+    let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
     let param_types = params
         .into_iter()
         .flat_map(|param_list| param_list.type_or_const_params())
@@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
                 let ty = param
                     .ty()
                     .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
-                    .unwrap_or_default();
+                    .unwrap_or_else(tt::Subtree::empty);
                 Some(ty)
             } else {
                 None
@@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
 fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
     let info = match parse_adt(tt) {
         Ok(info) => info,
-        Err(e) => return ExpandResult::only_err(e),
+        Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
     };
     let (params, args): (Vec<_>, Vec<_>) = info
         .param_types
@@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu
         .enumerate()
         .map(|(idx, param_ty)| {
             let ident = tt::Leaf::Ident(tt::Ident {
-                id: tt::TokenId::unspecified(),
+                span: tt::TokenId::unspecified(),
                 text: format!("T{idx}").into(),
             });
             let ident_ = ident.clone();
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs
index 5522bdf3b..9f3fa73d4 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin_fn_macro.rs
@@ -9,7 +9,9 @@ use syntax::{
     SmolStr,
 };
 
-use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
+use crate::{
+    db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
+};
 
 macro_rules! register_builtin {
     ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),*  ) => {
@@ -61,7 +63,7 @@ macro_rules! register_builtin {
     };
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct ExpandedEager {
     pub(crate) subtree: tt::Subtree,
     /// The included file ID of the include macro.
@@ -116,7 +118,7 @@ register_builtin! {
 }
 
 const DOLLAR_CRATE: tt::Ident =
-    tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() };
+    tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() };
 
 fn module_path_expand(
     _db: &dyn AstDatabase,
@@ -162,7 +164,7 @@ fn stringify_expand(
     _id: MacroCallId,
     tt: &tt::Subtree,
 ) -> ExpandResult<tt::Subtree> {
-    let pretty = tt::pretty(&tt.token_trees);
+    let pretty = ::tt::pretty(&tt.token_trees);
 
     let expanded = quote! {
         #pretty
@@ -194,11 +196,11 @@ fn assert_expand(
     let expanded = match &*args {
         [cond, panic_args @ ..] => {
             let comma = tt::Subtree {
-                delimiter: None,
+                delimiter: tt::Delimiter::unspecified(),
                 token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
                     char: ',',
                     spacing: tt::Spacing::Alone,
-                    id: tt::TokenId::unspecified(),
+                    span: tt::TokenId::unspecified(),
                 }))],
             };
             let cond = cond.clone();
@@ -247,7 +249,10 @@ fn format_args_expand(
     let mut args = parse_exprs_with_sep(tt, ',');
 
     if args.is_empty() {
-        return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into());
+        return ExpandResult::with_err(
+            tt::Subtree::empty(),
+            mbe::ExpandError::NoMatchingRule.into(),
+        );
     }
     for arg in &mut args {
         // Remove `key =`.
@@ -282,7 +287,7 @@ fn asm_expand(
     for tt in tt.token_trees.chunks(2) {
         match tt {
             [tt::TokenTree::Leaf(tt::Leaf::Literal(lit))]
-            | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] =>
+            | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] =>
             {
                 let krate = DOLLAR_CRATE.clone();
                 literals.push(quote!(#krate::format_args!(#lit);));
@@ -400,7 +405,7 @@ fn concat_expand(
         // FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses
         // to ensure the right parsing order, so skip the parentheses here. Ideally we'd
         // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623
-        if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t {
+        if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t {
             if let [tt] = &**token_trees {
                 if delim.kind == tt::DelimiterKind::Parenthesis {
                     t = tt;
@@ -459,9 +464,7 @@ fn concat_bytes_expand(
                 }
             }
             tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (),
-            tt::TokenTree::Subtree(tree)
-                if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) =>
-            {
+            tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => {
                 if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) {
                     err.get_or_insert(e);
                     break;
@@ -473,7 +476,7 @@ fn concat_bytes_expand(
             }
         }
     }
-    let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() };
+    let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
     ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
 }
 
@@ -521,7 +524,7 @@ fn concat_idents_expand(
             }
         }
     }
-    let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() };
+    let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
     ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
 }
 
@@ -572,7 +575,10 @@ fn include_expand(
         Ok((subtree, file_id)) => {
             ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) })
         }
-        Err(e) => ExpandResult::only_err(e),
+        Err(e) => ExpandResult::with_err(
+            ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
+            e,
+        ),
     }
 }
 
@@ -582,15 +588,18 @@ fn include_bytes_expand(
     tt: &tt::Subtree,
 ) -> ExpandResult<ExpandedEager> {
     if let Err(e) = parse_string(tt) {
-        return ExpandResult::only_err(e);
+        return ExpandResult::with_err(
+            ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
+            e,
+        );
     }
 
     // FIXME: actually read the file here if the user asked for macro expansion
     let res = tt::Subtree {
-        delimiter: None,
+        delimiter: tt::Delimiter::unspecified(),
         token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
             text: r#"b"""#.into(),
-            id: tt::TokenId::unspecified(),
+            span: tt::TokenId::unspecified(),
         }))],
     };
     ExpandResult::ok(ExpandedEager::new(res))
@@ -603,7 +612,12 @@ fn include_str_expand(
 ) -> ExpandResult<ExpandedEager> {
     let path = match parse_string(tt) {
         Ok(it) => it,
-        Err(e) => return ExpandResult::only_err(e),
+        Err(e) => {
+            return ExpandResult::with_err(
+                ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
+                e,
+            )
+        }
     };
 
     // FIXME: we're not able to read excluded files (which is most of them because
@@ -635,7 +649,12 @@ fn env_expand(
 ) -> ExpandResult<ExpandedEager> {
     let key = match parse_string(tt) {
         Ok(it) => it,
-        Err(e) => return ExpandResult::only_err(e),
+        Err(e) => {
+            return ExpandResult::with_err(
+                ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
+                e,
+            )
+        }
     };
 
     let mut err = None;
@@ -666,7 +685,12 @@ fn option_env_expand(
 ) -> ExpandResult<ExpandedEager> {
     let key = match parse_string(tt) {
         Ok(it) => it,
-        Err(e) => return ExpandResult::only_err(e),
+        Err(e) => {
+            return ExpandResult::with_err(
+                ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
+                e,
+            )
+        }
     };
 
     let expanded = match get_env_inner(db, arg_id, &key) {
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs
index b28e60187..76016274f 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs
@@ -14,7 +14,7 @@ use syntax::{
 
 use crate::{
     ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup,
-    hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
+    hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
     ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
     MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
 };
@@ -25,7 +25,7 @@ use crate::{
 /// an error will be emitted.
 ///
 /// Actual max for `analysis-stats .` at some point: 30672.
-static TOKEN_LIMIT: Limit = Limit::new(524_288);
+static TOKEN_LIMIT: Limit = Limit::new(1_048_576);
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum TokenExpander {
@@ -168,12 +168,14 @@ pub fn expand_speculative(
                 // Attributes may have an input token tree, build the subtree and map for this as well
                 // then try finding a token id for our token if it is inside this input subtree.
                 let item = ast::Item::cast(speculative_args.clone())?;
-                item.doc_comments_and_attrs().nth(invoc_attr_index as usize).and_then(Either::left)
+                item.doc_comments_and_attrs()
+                    .nth(invoc_attr_index.ast_index())
+                    .and_then(Either::left)
             }?;
             match attr.token_tree() {
                 Some(token_tree) => {
                     let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax());
-                    tree.delimiter = None;
+                    tree.delimiter = tt::Delimiter::unspecified();
 
                     let shift = mbe::Shift::new(&tt);
                     shift.shift_all(&mut tree);
@@ -208,7 +210,7 @@ pub fn expand_speculative(
     // Otherwise the expand query will fetch the non speculative attribute args and pass those instead.
     let mut speculative_expansion = match loc.def.kind {
         MacroDefKind::ProcMacro(expander, ..) => {
-            tt.delimiter = None;
+            tt.delimiter = tt::Delimiter::unspecified();
             expander.expand(db, loc.krate, &tt, attr_arg.as_ref())
         }
         MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => {
@@ -314,13 +316,13 @@ fn macro_arg(
 
     if loc.def.is_proc_macro() {
         // proc macros expect their inputs without parentheses, MBEs expect it with them included
-        tt.delimiter = None;
+        tt.delimiter = tt::Delimiter::unspecified();
     }
-
     Some(Arc::new((tt, tmap, fixups.undo_info)))
 }
 
 fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<SyntaxNode> {
+    // FIXME: handle `cfg_attr`
     (|| {
         let censor = match loc.kind {
             MacroCallKind::FnLike { .. } => return None,
@@ -328,7 +330,7 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<Sy
                 cov_mark::hit!(derive_censoring);
                 ast::Item::cast(node.clone())?
                     .attrs()
-                    .take(derive_attr_index as usize + 1)
+                    .take(derive_attr_index.ast_index() + 1)
                     // FIXME, this resolution should not be done syntactically
                     // derive is a proper macro now, no longer builtin
                     // But we do not have resolution at this stage, this means
@@ -343,7 +345,7 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<Sy
                 cov_mark::hit!(attribute_macro_attr_censoring);
                 ast::Item::cast(node.clone())?
                     .doc_comments_and_attrs()
-                    .nth(invoc_attr_index as usize)
+                    .nth(invoc_attr_index.ast_index())
                     .and_then(Either::left)
                     .map(|attr| attr.syntax().clone())
                     .into_iter()
@@ -476,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult<tt::
     let macro_arg = match db.macro_arg(id) {
         Some(it) => it,
         None => {
-            return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into()))
+            return ExpandResult::with_err(
+                tt::Subtree::empty(),
+                ExpandError::Other("No arguments for proc-macro".into()),
+            )
         }
     };
 
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs
index a1474c44e..dfab7ec92 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs
@@ -108,7 +108,7 @@ pub fn expand_eager_macro(
         .value
         .token_tree()
         .map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
-        .unwrap_or_default();
+        .unwrap_or_else(tt::Subtree::empty);
 
     let ast_map = db.ast_id_map(macro_call.file_id);
     let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@@ -165,9 +165,9 @@ pub fn expand_eager_macro(
     }
 }
 
-fn to_subtree(node: &SyntaxNode) -> tt::Subtree {
+fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree {
     let mut subtree = mbe::syntax_node_to_token_tree(node).0;
-    subtree.delimiter = None;
+    subtree.delimiter = crate::tt::Delimiter::unspecified();
     subtree
 }
 
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs
index 75d364d5f..c811d1c66 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs
@@ -9,7 +9,7 @@ use syntax::{
     ast::{self, AstNode, HasLoopBody},
     match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
 };
-use tt::Subtree;
+use tt::token_id::Subtree;
 
 /// The result of calculating fixes for a syntax node -- a bunch of changes
 /// (appending to and replacing nodes), the information that is needed to
@@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups(
     tt.token_trees = tts
         .into_iter()
         .filter(|tt| match tt {
-            tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID),
+            tt::TokenTree::Leaf(leaf) => {
+                token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID)
+            }
             tt::TokenTree::Subtree(st) => {
-                st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID))
+                token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID)
             }
         })
         .flat_map(|tt| match tt {
@@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups(
                 SmallVec::from_const([tt.into()])
             }
             tt::TokenTree::Leaf(leaf) => {
-                if let Some(id) = token_map.synthetic_token_id(leaf.id()) {
+                if let Some(id) = token_map.synthetic_token_id(*leaf.span()) {
                     let original = undo_info.original[id.0 as usize].clone();
-                    if original.delimiter.is_none() {
+                    if original.delimiter.kind == tt::DelimiterKind::Invisible {
                         original.token_trees.into()
                     } else {
                         SmallVec::from_const([original.into()])
@@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups(
 mod tests {
     use expect_test::{expect, Expect};
 
+    use crate::tt;
+
     use super::reverse_fixups;
 
     // The following three functions are only meant to check partial structural equivalence of
@@ -341,7 +345,7 @@ mod tests {
     }
 
     fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
-        a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind)
+        a.delimiter.kind == b.delimiter.kind
             && a.token_trees.len() == b.token_trees.len()
             && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
     }
@@ -386,7 +390,7 @@ mod tests {
         let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
         assert!(
             check_subtree_eq(&tt, &original_as_tt),
-            "different token tree: {tt:?}, {original_as_tt:?}"
+            "different token tree: {tt:?},\n{original_as_tt:?}"
         );
     }
 
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs b/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs
index df1e20256..2300ee9d0 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs
@@ -128,7 +128,7 @@ struct HygieneInfo {
     attr_input_or_mac_def_start: Option<InFile<TextSize>>,
 
     macro_def: Arc<TokenExpander>,
-    macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
+    macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
     macro_arg_shift: mbe::Shift,
     exp_map: Arc<mbe::TokenMap>,
 }
@@ -191,7 +191,7 @@ fn make_hygiene_info(
             let tt = ast_id
                 .to_node(db)
                 .doc_comments_and_attrs()
-                .nth(invoc_attr_index as usize)
+                .nth(invoc_attr_index.ast_index())
                 .and_then(Either::left)?
                 .token_tree()?;
             Some(InFile::new(ast_id.file_id, tt))
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs b/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs
index bc5f9f3b8..a52716cc0 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/lib.rs
@@ -17,10 +17,13 @@ pub mod proc_macro;
 pub mod quote;
 pub mod eager;
 pub mod mod_path;
+pub mod attrs;
 mod fixup;
 
 pub use mbe::{Origin, ValueResult};
 
+use ::tt::token_id as tt;
+
 use std::{fmt, hash::Hash, iter, sync::Arc};
 
 use base_db::{
@@ -37,6 +40,7 @@ use syntax::{
 
 use crate::{
     ast_id_map::FileAstId,
+    attrs::AttrId,
     builtin_attr_macro::BuiltinAttrExpander,
     builtin_derive_macro::BuiltinDeriveExpander,
     builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander},
@@ -51,6 +55,7 @@ pub type ExpandResult<T> = ValueResult<T, ExpandError>;
 pub enum ExpandError {
     UnresolvedProcMacro(CrateId),
     Mbe(mbe::ExpandError),
+    RecursionOverflowPosioned,
     Other(Box<str>),
 }
 
@@ -65,6 +70,9 @@ impl fmt::Display for ExpandError {
         match self {
             ExpandError::UnresolvedProcMacro(_) => f.write_str("unresolved proc-macro"),
             ExpandError::Mbe(it) => it.fmt(f),
+            ExpandError::RecursionOverflowPosioned => {
+                f.write_str("overflow expanding the original macro")
+            }
             ExpandError::Other(it) => f.write_str(it),
         }
     }
@@ -114,6 +122,7 @@ pub struct MacroDefId {
     pub krate: CrateId,
     pub kind: MacroDefKind,
     pub local_inner: bool,
+    pub allow_internal_unsafe: bool,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -145,7 +154,7 @@ pub enum MacroCallKind {
         ///
         /// Outer attributes are counted first, then inner attributes. This does not support
         /// out-of-line modules, which may have attributes spread across 2 files!
-        derive_attr_index: u32,
+        derive_attr_index: AttrId,
         /// Index of the derive macro in the derive attribute
         derive_index: u32,
     },
@@ -156,7 +165,7 @@ pub enum MacroCallKind {
         ///
         /// Outer attributes are counted first, then inner attributes. This does not support
         /// out-of-line modules, which may have attributes spread across 2 files!
-        invoc_attr_index: u32,
+        invoc_attr_index: AttrId,
         /// Whether this attribute is the `#[derive]` attribute.
         is_derive: bool,
     },
@@ -261,10 +270,11 @@ impl HirFileId {
         });
         let attr_input_or_mac_def = def.or_else(|| match loc.kind {
             MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => {
+                // FIXME: handle `cfg_attr`
                 let tt = ast_id
                     .to_node(db)
                     .doc_comments_and_attrs()
-                    .nth(invoc_attr_index as usize)
+                    .nth(invoc_attr_index.ast_index())
                     .and_then(Either::left)?
                     .token_tree()?;
                 Some(InFile::new(ast_id.file_id, tt))
@@ -353,6 +363,14 @@ impl HirFileId {
         }
     }
 
+    #[inline]
+    pub fn file_id(self) -> Option<FileId> {
+        match self.0 & Self::MACRO_FILE_TAG_MASK {
+            0 => Some(FileId(self.0)),
+            _ => None,
+        }
+    }
+
     fn repr(self) -> HirFileIdRepr {
         match self.0 & Self::MACRO_FILE_TAG_MASK {
             0 => HirFileIdRepr::FileId(FileId(self.0)),
@@ -397,8 +415,7 @@ impl MacroDefId {
     }
 }
 
-// FIXME: attribute indices do not account for `cfg_attr`, which means that we'll strip the whole
-// `cfg_attr` instead of just one of the attributes it expands to
+// FIXME: attribute indices do not account for nested `cfg_attr`
 
 impl MacroCallKind {
     /// Returns the file containing the macro invocation.
@@ -419,7 +436,7 @@ impl MacroCallKind {
                 // FIXME: handle `cfg_attr`
                 ast_id.with_value(ast_id.to_node(db)).map(|it| {
                     it.doc_comments_and_attrs()
-                        .nth(*derive_attr_index as usize)
+                        .nth(derive_attr_index.ast_index())
                         .and_then(|it| match it {
                             Either::Left(attr) => Some(attr.syntax().clone()),
                             Either::Right(_) => None,
@@ -431,7 +448,7 @@ impl MacroCallKind {
                 // FIXME: handle `cfg_attr`
                 ast_id.with_value(ast_id.to_node(db)).map(|it| {
                     it.doc_comments_and_attrs()
-                        .nth(*invoc_attr_index as usize)
+                        .nth(invoc_attr_index.ast_index())
                         .and_then(|it| match it {
                             Either::Left(attr) => Some(attr.syntax().clone()),
                             Either::Right(_) => None,
@@ -488,19 +505,21 @@ impl MacroCallKind {
             MacroCallKind::FnLike { ast_id, .. } => ast_id.to_node(db).syntax().text_range(),
             MacroCallKind::Derive { ast_id, derive_attr_index, .. } => {
                 // FIXME: should be the range of the macro name, not the whole derive
+                // FIXME: handle `cfg_attr`
                 ast_id
                     .to_node(db)
                     .doc_comments_and_attrs()
-                    .nth(derive_attr_index as usize)
+                    .nth(derive_attr_index.ast_index())
                     .expect("missing derive")
                     .expect_left("derive is a doc comment?")
                     .syntax()
                     .text_range()
             }
+            // FIXME: handle `cfg_attr`
             MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => ast_id
                 .to_node(db)
                 .doc_comments_and_attrs()
-                .nth(invoc_attr_index as usize)
+                .nth(invoc_attr_index.ast_index())
                 .expect("missing attribute")
                 .expect_left("attribute macro is a doc comment?")
                 .syntax()
@@ -592,9 +611,10 @@ impl ExpansionInfo {
             let token_range = token.value.text_range();
             match &loc.kind {
                 MacroCallKind::Attr { attr_args, invoc_attr_index, is_derive, .. } => {
+                    // FIXME: handle `cfg_attr`
                     let attr = item
                         .doc_comments_and_attrs()
-                        .nth(*invoc_attr_index as usize)
+                        .nth(invoc_attr_index.ast_index())
                         .and_then(Either::left)?;
                     match attr.token_tree() {
                         Some(token_tree)
@@ -1031,3 +1051,5 @@ impl ExpandTo {
 pub struct UnresolvedMacro {
     pub path: ModPath,
 }
+
+intern::impl_internable!(ModPath, attrs::AttrInput);
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs
index e8b3e312a..c3462beac 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/name.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/name.rs
@@ -2,7 +2,7 @@
 
 use std::fmt;
 
-use syntax::{ast, SmolStr, SyntaxKind};
+use syntax::{ast, utils::is_raw_identifier, SmolStr};
 
 /// `Name` is a wrapper around string, which is used in hir for both references
 /// and declarations. In theory, names should also carry hygiene info, but we are
@@ -33,11 +33,6 @@ impl fmt::Display for Name {
     }
 }
 
-fn is_raw_identifier(name: &str) -> bool {
-    let is_keyword = SyntaxKind::from_keyword(name).is_some();
-    is_keyword && !matches!(name, "self" | "crate" | "super" | "Self")
-}
-
 impl<'a> fmt::Display for UnescapedName<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match &self.0 .0 {
@@ -133,6 +128,14 @@ impl Name {
         }
     }
 
+    /// Returns the text this name represents if it isn't a tuple field.
+    pub fn as_str(&self) -> Option<&str> {
+        match &self.0 {
+            Repr::Text(it) => Some(it),
+            _ => None,
+        }
+    }
+
     /// Returns the textual representation of this name as a [`SmolStr`].
     /// Prefer using this over [`ToString::to_string`] if possible as this conversion is cheaper in
     /// the general case.
@@ -183,7 +186,7 @@ impl AsName for ast::NameOrNameRef {
     }
 }
 
-impl AsName for tt::Ident {
+impl<Span> AsName for tt::Ident<Span> {
     fn as_name(&self) -> Name {
         Name::resolve(&self.text)
     }
@@ -339,6 +342,7 @@ pub mod known {
         recursion_limit,
         feature,
         // known methods of lang items
+        call_once,
         eq,
         ne,
         ge,
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs
index 5afdcc0e6..3f4d2540c 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/proc_macro.rs
@@ -3,7 +3,7 @@
 use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
 use stdx::never;
 
-use crate::{db::AstDatabase, ExpandError, ExpandResult};
+use crate::{db::AstDatabase, tt, ExpandError, ExpandResult};
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub struct ProcMacroExpander {
@@ -39,7 +39,10 @@ impl ProcMacroExpander {
                     Ok(proc_macros) => proc_macros,
                     Err(_) => {
                         never!("Non-dummy expander even though there are no proc macros");
-                        return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
+                        return ExpandResult::with_err(
+                            tt::Subtree::empty(),
+                            ExpandError::Other("Internal error".into()),
+                        );
                     }
                 };
                 let proc_macro = match proc_macros.get(id.0 as usize) {
@@ -50,7 +53,10 @@ impl ProcMacroExpander {
                             proc_macros.len(),
                             id.0
                         );
-                        return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
+                        return ExpandResult::with_err(
+                            tt::Subtree::empty(),
+                            ExpandError::Other("Internal error".into()),
+                        );
                     }
                 };
 
@@ -69,13 +75,17 @@ impl ProcMacroExpander {
                             }
                         }
                         ProcMacroExpansionError::System(text)
-                        | ProcMacroExpansionError::Panic(text) => {
-                            ExpandResult::only_err(ExpandError::Other(text.into()))
-                        }
+                        | ProcMacroExpansionError::Panic(text) => ExpandResult::with_err(
+                            tt::Subtree::empty(),
+                            ExpandError::Other(text.into()),
+                        ),
                     },
                 }
             }
-            None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)),
+            None => ExpandResult::with_err(
+                tt::Subtree::empty(),
+                ExpandError::UnresolvedProcMacro(self.krate),
+            ),
         }
     }
 }
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs
index c0a7bc7ca..63586f9da 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/quote.rs
@@ -9,17 +9,18 @@
 #[macro_export]
 macro_rules! __quote {
     () => {
-        Vec::<tt::TokenTree>::new()
+        Vec::<crate::tt::TokenTree>::new()
     };
 
     ( @SUBTREE $delim:ident $($tt:tt)* ) => {
         {
             let children = $crate::__quote!($($tt)*);
-            tt::Subtree {
-                delimiter: Some(tt::Delimiter {
-                    kind: tt::DelimiterKind::$delim,
-                    id: tt::TokenId::unspecified(),
-                }),
+            crate::tt::Subtree {
+                delimiter: crate::tt::Delimiter {
+                    kind: crate::tt::DelimiterKind::$delim,
+                    open: crate::tt::TokenId::unspecified(),
+                    close: crate::tt::TokenId::unspecified(),
+                },
                 token_trees: $crate::quote::IntoTt::to_tokens(children),
             }
         }
@@ -28,10 +29,10 @@ macro_rules! __quote {
     ( @PUNCT $first:literal ) => {
         {
             vec![
-                tt::Leaf::Punct(tt::Punct {
+                crate::tt::Leaf::Punct(crate::tt::Punct {
                     char: $first,
-                    spacing: tt::Spacing::Alone,
-                    id: tt::TokenId::unspecified(),
+                    spacing: crate::tt::Spacing::Alone,
+                    span: crate::tt::TokenId::unspecified(),
                 }).into()
             ]
         }
@@ -40,15 +41,15 @@ macro_rules! __quote {
     ( @PUNCT $first:literal, $sec:literal ) => {
         {
             vec![
-                tt::Leaf::Punct(tt::Punct {
+                crate::tt::Leaf::Punct(crate::tt::Punct {
                     char: $first,
-                    spacing: tt::Spacing::Joint,
-                    id: tt::TokenId::unspecified(),
+                    spacing: crate::tt::Spacing::Joint,
+                    span: crate::tt::TokenId::unspecified(),
                 }).into(),
-                tt::Leaf::Punct(tt::Punct {
+                crate::tt::Leaf::Punct(crate::tt::Punct {
                     char: $sec,
-                    spacing: tt::Spacing::Alone,
-                    id: tt::TokenId::unspecified(),
+                    spacing: crate::tt::Spacing::Alone,
+                    span: crate::tt::TokenId::unspecified(),
                 }).into()
             ]
         }
@@ -67,7 +68,7 @@ macro_rules! __quote {
 
     ( ## $first:ident $($tail:tt)* ) => {
         {
-            let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<tt::TokenTree>>();
+            let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<crate::tt::TokenTree>>();
             let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
             tokens.append(&mut tail_tokens);
             tokens
@@ -86,9 +87,9 @@ macro_rules! __quote {
     // Ident
     ( $tt:ident ) => {
         vec![ {
-            tt::Leaf::Ident(tt::Ident {
+            crate::tt::Leaf::Ident(crate::tt::Ident {
                 text: stringify!($tt).into(),
-                id: tt::TokenId::unspecified(),
+                span: crate::tt::TokenId::unspecified(),
             }).into()
         }]
     };
@@ -127,42 +128,42 @@ macro_rules! quote {
 }
 
 pub(crate) trait IntoTt {
-    fn to_subtree(self) -> tt::Subtree;
-    fn to_tokens(self) -> Vec<tt::TokenTree>;
+    fn to_subtree(self) -> crate::tt::Subtree;
+    fn to_tokens(self) -> Vec<crate::tt::TokenTree>;
 }
 
-impl IntoTt for Vec<tt::TokenTree> {
-    fn to_subtree(self) -> tt::Subtree {
-        tt::Subtree { delimiter: None, token_trees: self }
+impl IntoTt for Vec<crate::tt::TokenTree> {
+    fn to_subtree(self) -> crate::tt::Subtree {
+        crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self }
     }
 
-    fn to_tokens(self) -> Vec<tt::TokenTree> {
+    fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
         self
     }
 }
 
-impl IntoTt for tt::Subtree {
-    fn to_subtree(self) -> tt::Subtree {
+impl IntoTt for crate::tt::Subtree {
+    fn to_subtree(self) -> crate::tt::Subtree {
         self
     }
 
-    fn to_tokens(self) -> Vec<tt::TokenTree> {
-        vec![tt::TokenTree::Subtree(self)]
+    fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
+        vec![crate::tt::TokenTree::Subtree(self)]
     }
 }
 
 pub(crate) trait ToTokenTree {
-    fn to_token(self) -> tt::TokenTree;
+    fn to_token(self) -> crate::tt::TokenTree;
 }
 
-impl ToTokenTree for tt::TokenTree {
-    fn to_token(self) -> tt::TokenTree {
+impl ToTokenTree for crate::tt::TokenTree {
+    fn to_token(self) -> crate::tt::TokenTree {
         self
     }
 }
 
-impl ToTokenTree for tt::Subtree {
-    fn to_token(self) -> tt::TokenTree {
+impl ToTokenTree for crate::tt::Subtree {
+    fn to_token(self) -> crate::tt::TokenTree {
         self.into()
     }
 }
@@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees {
     ($($ty:ty => $this:ident $im:block);*) => {
         $(
             impl ToTokenTree for $ty {
-                fn to_token($this) -> tt::TokenTree {
-                    let leaf: tt::Leaf = $im.into();
+                fn to_token($this) -> crate::tt::TokenTree {
+                    let leaf: crate::tt::Leaf = $im.into();
                     leaf.into()
                 }
             }
 
             impl ToTokenTree for &$ty {
-                fn to_token($this) -> tt::TokenTree {
-                    let leaf: tt::Leaf = $im.clone().into();
+                fn to_token($this) -> crate::tt::TokenTree {
+                    let leaf: crate::tt::Leaf = $im.clone().into();
                     leaf.into()
                 }
             }
@@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees {
 }
 
 impl_to_to_tokentrees! {
-    u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
-    usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
-    i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
-    bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
-    tt::Leaf => self { self };
-    tt::Literal => self { self };
-    tt::Ident => self { self };
-    tt::Punct => self { self };
-    &str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}};
-    String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}
+    u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
+    usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
+    i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
+    bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
+    crate::tt::Leaf => self { self };
+    crate::tt::Literal => self { self };
+    crate::tt::Ident => self { self };
+    crate::tt::Punct => self { self };
+    &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}};
+    String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}
 }
 
 #[cfg(test)]
@@ -223,8 +224,8 @@ mod tests {
         assert_eq!(quote!(#s).to_string(), "\"hello\"");
     }
 
-    fn mk_ident(name: &str) -> tt::Ident {
-        tt::Ident { text: name.into(), id: tt::TokenId::unspecified() }
+    fn mk_ident(name: &str) -> crate::tt::Ident {
+        crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() }
     }
 
     #[test]
@@ -234,7 +235,7 @@ mod tests {
         let quoted = quote!(#a);
         assert_eq!(quoted.to_string(), "hello");
         let t = format!("{quoted:?}");
-        assert_eq!(t, "SUBTREE $\n  IDENT   hello 4294967295");
+        assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n  IDENT   hello 4294967295");
     }
 
     #[test]
@@ -263,11 +264,12 @@ mod tests {
         let fields = [mk_ident("name"), mk_ident("id")];
         let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees);
 
-        let list = tt::Subtree {
-            delimiter: Some(tt::Delimiter {
-                kind: tt::DelimiterKind::Brace,
-                id: tt::TokenId::unspecified(),
-            }),
+        let list = crate::tt::Subtree {
+            delimiter: crate::tt::Delimiter {
+                kind: crate::tt::DelimiterKind::Brace,
+                open: crate::tt::TokenId::unspecified(),
+                close: crate::tt::TokenId::unspecified(),
+            },
             token_trees: fields.collect(),
         };