From 9918693037dce8aa4bb6f08741b6812923486c18 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 19 Jun 2024 11:26:03 +0200 Subject: Merging upstream version 1.76.0+dfsg1. Signed-off-by: Daniel Baumann --- src/tools/rust-analyzer/crates/tt/Cargo.toml | 1 + src/tools/rust-analyzer/crates/tt/src/lib.rs | 264 +++++++++++++++------------ 2 files changed, 151 insertions(+), 114 deletions(-) (limited to 'src/tools/rust-analyzer/crates/tt') diff --git a/src/tools/rust-analyzer/crates/tt/Cargo.toml b/src/tools/rust-analyzer/crates/tt/Cargo.toml index a28ee5f1c..572224497 100644 --- a/src/tools/rust-analyzer/crates/tt/Cargo.toml +++ b/src/tools/rust-analyzer/crates/tt/Cargo.toml @@ -13,5 +13,6 @@ doctest = false [dependencies] smol_str.workspace = true +text-size.workspace = true stdx.workspace = true diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs index b5a72bec0..481d57540 100644 --- a/src/tools/rust-analyzer/crates/tt/src/lib.rs +++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs @@ -2,115 +2,98 @@ //! input and output) of macros. It closely mirrors `proc_macro` crate's //! `TokenTree`. -#![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] +#![warn(rust_2018_idioms, unused_lifetimes)] use std::fmt; use stdx::impl_from; pub use smol_str::SmolStr; +pub use text_size::{TextRange, TextSize}; + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct SpanData { + /// The text range of this span, relative to the anchor. + /// We need the anchor for incrementality, as storing absolute ranges will require + /// recomputation on every change in a file at all times. + pub range: TextRange, + pub anchor: Anchor, + /// The syntax context of the span. + pub ctx: Ctx, +} -/// Represents identity of the token. -/// -/// For hygiene purposes, we need to track which expanded tokens originated from -/// which source tokens. We do it by assigning an distinct identity to each -/// source token and making sure that identities are preserved during macro -/// expansion. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct TokenId(pub u32); +impl Span for SpanData { + #[allow(deprecated)] + const DUMMY: Self = SpanData { + range: TextRange::empty(TextSize::new(0)), + anchor: Anchor::DUMMY, + ctx: Ctx::DUMMY, + }; +} -impl fmt::Debug for TokenId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } +pub trait Span: std::fmt::Debug + Copy + Sized + Eq { + // FIXME: Should not exist. Dummy spans will always be wrong if they leak somewhere. Instead, + // the call site or def site spans should be used in relevant places, its just that we don't + // expose those everywhere in the yet. + const DUMMY: Self; } -impl TokenId { - pub const UNSPECIFIED: TokenId = TokenId(!0); - pub const fn unspecified() -> TokenId { - Self::UNSPECIFIED - } +// FIXME: Should not exist +pub trait SpanAnchor: + std::fmt::Debug + Copy + Sized + Eq + Copy + fmt::Debug + std::hash::Hash +{ + #[deprecated(note = "this should not exist")] + const DUMMY: Self; } -pub mod token_id { - pub use crate::{DelimiterKind, Spacing, TokenId}; - pub type Span = crate::TokenId; - pub type Subtree = crate::Subtree; - pub type Punct = crate::Punct; - pub type Delimiter = crate::Delimiter; - pub type Leaf = crate::Leaf; - pub type Ident = crate::Ident; - pub type Literal = crate::Literal; - pub type TokenTree = crate::TokenTree; - pub mod buffer { - pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>; - pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>; - pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>; - } +// FIXME: Should not exist +pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { + #[deprecated(note = "this should not exist")] + const DUMMY: Self; +} - impl Delimiter { - pub const UNSPECIFIED: Self = Self { - open: TokenId::UNSPECIFIED, - close: TokenId::UNSPECIFIED, - kind: DelimiterKind::Invisible, - }; - pub const fn unspecified() -> Self { - Self::UNSPECIFIED - } - } - impl Subtree { - pub const fn empty() -> Self { - Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } - } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), +} +impl_from!(Leaf, Subtree for TokenTree); +impl TokenTree { + pub const fn empty(span: S) -> Self { + Self::Subtree(Subtree { + delimiter: Delimiter::invisible_spanned(span), + token_trees: vec![], + }) } - impl TokenTree { - pub const fn empty() -> Self { - Self::Subtree(Subtree::empty()) + + pub fn subtree_or_wrap(self) -> Subtree { + match self { + TokenTree::Leaf(_) => { + Subtree { delimiter: Delimiter::DUMMY_INVISIBLE, token_trees: vec![self] } + } + TokenTree::Subtree(s) => s, } } - - impl Subtree { - pub fn visit_ids(&mut self, f: &mut impl FnMut(TokenId) -> TokenId) { - self.delimiter.open = f(self.delimiter.open); - self.delimiter.close = f(self.delimiter.close); - self.token_trees.iter_mut().for_each(|tt| match tt { - crate::TokenTree::Leaf(leaf) => match leaf { - crate::Leaf::Literal(it) => it.span = f(it.span), - crate::Leaf::Punct(it) => it.span = f(it.span), - crate::Leaf::Ident(it) => it.span = f(it.span), - }, - crate::TokenTree::Subtree(s) => s.visit_ids(f), - }) + pub fn subtree_or_wrap2(self, span: DelimSpan) -> Subtree { + match self { + TokenTree::Leaf(_) => Subtree { + delimiter: Delimiter::invisible_delim_spanned(span), + token_trees: vec![self], + }, + TokenTree::Subtree(s) => s, } } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct SyntaxContext(pub u32); - -// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -// pub struct Span { -// pub id: TokenId, -// pub ctx: SyntaxContext, -// } -// pub type Span = (TokenId, SyntaxContext); - #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), } -impl_from!(Leaf, Subtree for TokenTree); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), -} - -impl Leaf { - pub fn span(&self) -> &Span { +impl Leaf { + pub fn span(&self) -> &S { match self { Leaf::Literal(it) => &it.span, Leaf::Punct(it) => &it.span, @@ -118,21 +101,74 @@ impl Leaf { } } } -impl_from!(Literal, Punct, Ident for Leaf); +impl_from!(Literal, Punct, Ident for Leaf); #[derive(Clone, PartialEq, Eq, Hash)] -pub struct Subtree { - pub delimiter: Delimiter, - pub token_trees: Vec>, +pub struct Subtree { + pub delimiter: Delimiter, + pub token_trees: Vec>, +} + +impl Subtree { + pub const fn empty(span: DelimSpan) -> Self { + Subtree { delimiter: Delimiter::invisible_delim_spanned(span), token_trees: vec![] } + } + + pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) { + self.delimiter.open = f(self.delimiter.open); + self.delimiter.close = f(self.delimiter.close); + self.token_trees.iter_mut().for_each(|tt| match tt { + crate::TokenTree::Leaf(leaf) => match leaf { + crate::Leaf::Literal(it) => it.span = f(it.span), + crate::Leaf::Punct(it) => it.span = f(it.span), + crate::Leaf::Ident(it) => it.span = f(it.span), + }, + crate::TokenTree::Subtree(s) => s.visit_ids(f), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct DelimSpan { + pub open: S, + pub close: S, +} + +impl DelimSpan { + // FIXME should not exist + pub const DUMMY: Self = Self { open: S::DUMMY, close: S::DUMMY }; } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub open: Span, - pub close: Span, +pub struct Delimiter { + pub open: S, + pub close: S, pub kind: DelimiterKind, } +impl Delimiter { + // FIXME should not exist + pub const DUMMY_INVISIBLE: Self = + Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible }; + + // FIXME should not exist + pub const fn dummy_invisible() -> Self { + Self::DUMMY_INVISIBLE + } + + pub const fn invisible_spanned(span: S) -> Self { + Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + } + + pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { + Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + } + + pub fn delim_span(&self) -> DelimSpan { + DelimSpan { open: self.open, close: self.close } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum DelimiterKind { Parenthesis, @@ -142,16 +178,16 @@ pub enum DelimiterKind { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { +pub struct Literal { pub text: SmolStr, - pub span: Span, + pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { +pub struct Punct { pub char: char, pub spacing: Spacing, - pub span: Span, + pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -162,9 +198,9 @@ pub enum Spacing { #[derive(Debug, Clone, PartialEq, Eq, Hash)] /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. -pub struct Ident { +pub struct Ident { pub text: SmolStr, - pub span: Span, + pub span: S, } impl Ident { @@ -173,9 +209,9 @@ impl Ident { } } -fn print_debug_subtree( +fn print_debug_subtree( f: &mut fmt::Formatter<'_>, - subtree: &Subtree, + subtree: &Subtree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); @@ -203,9 +239,9 @@ fn print_debug_subtree( Ok(()) } -fn print_debug_token( +fn print_debug_token( f: &mut fmt::Formatter<'_>, - tkn: &TokenTree, + tkn: &TokenTree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); @@ -231,13 +267,13 @@ fn print_debug_token( Ok(()) } -impl fmt::Debug for Subtree { +impl fmt::Debug for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { print_debug_subtree(f, self, 0) } } -impl fmt::Display for TokenTree { +impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { TokenTree::Leaf(it) => fmt::Display::fmt(it, f), @@ -246,7 +282,7 @@ impl fmt::Display for TokenTree { } } -impl fmt::Display for Subtree { +impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let (l, r) = match self.delimiter.kind { DelimiterKind::Parenthesis => ("(", ")"), @@ -274,7 +310,7 @@ impl fmt::Display for Subtree { } } -impl fmt::Display for Leaf { +impl fmt::Display for Leaf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Leaf::Ident(it) => fmt::Display::fmt(it, f), @@ -284,25 +320,25 @@ impl fmt::Display for Leaf { } } -impl fmt::Display for Ident { +impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Literal { +impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Punct { +impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.char, f) } } -impl Subtree { +impl Subtree { /// Count the number of tokens recursively pub fn count(&self) -> usize { let children_count = self @@ -318,7 +354,7 @@ impl Subtree { } } -impl Subtree { +impl Subtree { /// A simple line string used for debugging pub fn as_debug_string(&self) -> String { let delim = match self.delimiter.kind { @@ -366,8 +402,8 @@ impl Subtree { pub mod buffer; -pub fn pretty(tkns: &[TokenTree]) -> String { - fn tokentree_to_text(tkn: &TokenTree) -> String { +pub fn pretty(tkns: &[TokenTree]) -> String { + fn tokentree_to_text(tkn: &TokenTree) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(), TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), -- cgit v1.2.3