summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_expand
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:11:38 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:12:43 +0000
commitcf94bdc0742c13e2a0cac864c478b8626b266e1b (patch)
tree044670aa50cc5e2b4229aa0b6b3df6676730c0a6 /compiler/rustc_expand
parentAdding debian version 1.65.0+dfsg1-2. (diff)
downloadrustc-cf94bdc0742c13e2a0cac864c478b8626b266e1b.tar.xz
rustc-cf94bdc0742c13e2a0cac864c478b8626b266e1b.zip
Merging upstream version 1.66.0+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'compiler/rustc_expand')
-rw-r--r--compiler/rustc_expand/src/base.rs59
-rw-r--r--compiler/rustc_expand/src/build.rs4
-rw-r--r--compiler/rustc_expand/src/config.rs10
-rw-r--r--compiler/rustc_expand/src/errors.rs26
-rw-r--r--compiler/rustc_expand/src/expand.rs72
-rw-r--r--compiler/rustc_expand/src/lib.rs1
-rw-r--r--compiler/rustc_expand/src/mbe/macro_rules.rs26
-rw-r--r--compiler/rustc_expand/src/placeholders.rs8
-rw-r--r--compiler/rustc_expand/src/proc_macro_server.rs146
-rw-r--r--compiler/rustc_expand/src/tokenstream/tests.rs18
10 files changed, 246 insertions, 124 deletions
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index e1da3ecde..c8de60ccb 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -10,17 +10,19 @@ use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind
use rustc_attr::{self as attr, Deprecation, Stability};
use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
use rustc_data_structures::sync::{self, Lrc};
-use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan, PResult};
+use rustc_errors::{
+ Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, MultiSpan, PResult,
+};
use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics};
use rustc_parse::{self, parser, MACRO_ARGUMENTS};
-use rustc_session::{parse::ParseSess, Limit, Session, SessionDiagnostic};
+use rustc_session::{parse::ParseSess, Limit, Session};
use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
use rustc_span::edition::Edition;
use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId};
use rustc_span::source_map::SourceMap;
use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::{FileName, Span, DUMMY_SP};
+use rustc_span::{BytePos, FileName, RealFileName, Span, DUMMY_SP};
use smallvec::{smallvec, SmallVec};
use std::default::Default;
@@ -1109,12 +1111,12 @@ impl<'a> ExtCtxt<'a> {
pub fn create_err(
&self,
- err: impl SessionDiagnostic<'a>,
+ err: impl IntoDiagnostic<'a>,
) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
self.sess.create_err(err)
}
- pub fn emit_err(&self, err: impl SessionDiagnostic<'a>) -> ErrorGuaranteed {
+ pub fn emit_err(&self, err: impl IntoDiagnostic<'a>) -> ErrorGuaranteed {
self.sess.emit_err(err)
}
@@ -1226,8 +1228,9 @@ pub fn expr_to_spanned_string<'a>(
ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)),
ast::LitKind::ByteStr(_) => {
let mut err = cx.struct_span_err(l.span, err_msg);
+ let span = expr.span.shrink_to_lo();
err.span_suggestion(
- expr.span.shrink_to_lo(),
+ span.with_hi(span.lo() + BytePos(1)),
"consider removing the leading `b`",
"",
Applicability::MaybeIncorrect,
@@ -1420,16 +1423,40 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &ParseSess) -> bool {
if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
if let [variant] = &*enum_def.variants {
if variant.ident.name == sym::Input {
- sess.buffer_lint_with_diagnostic(
- &PROC_MACRO_BACK_COMPAT,
- item.ident.span,
- ast::CRATE_NODE_ID,
- "using `procedural-masquerade` crate",
- BuiltinLintDiagnostics::ProcMacroBackCompat(
- "The `procedural-masquerade` crate has been unnecessary since Rust 1.30.0. \
- Versions of this crate below 0.1.7 will eventually stop compiling.".to_string())
- );
- return true;
+ let filename = sess.source_map().span_to_filename(item.ident.span);
+ if let FileName::Real(RealFileName::LocalPath(path)) = filename {
+ if let Some(c) = path
+ .components()
+ .flat_map(|c| c.as_os_str().to_str())
+ .find(|c| c.starts_with("rental") || c.starts_with("allsorts-rental"))
+ {
+ let crate_matches = if c.starts_with("allsorts-rental") {
+ true
+ } else {
+ let mut version = c.trim_start_matches("rental-").split(".");
+ version.next() == Some("0")
+ && version.next() == Some("5")
+ && version
+ .next()
+ .and_then(|c| c.parse::<u32>().ok())
+ .map_or(false, |v| v < 6)
+ };
+
+ if crate_matches {
+ sess.buffer_lint_with_diagnostic(
+ &PROC_MACRO_BACK_COMPAT,
+ item.ident.span,
+ ast::CRATE_NODE_ID,
+ "using an old version of `rental`",
+ BuiltinLintDiagnostics::ProcMacroBackCompat(
+ "older versions of the `rental` crate will stop compiling in future versions of Rust; \
+ please update to `rental` v0.5.6, or switch to one of the `rental` alternatives".to_string()
+ )
+ );
+ return true;
+ }
+ }
+ }
}
}
}
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 50d2be3ce..0952e65cf 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -252,6 +252,10 @@ impl<'a> ExtCtxt<'a> {
self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower))
}
+ pub fn expr_field(&self, span: Span, expr: P<Expr>, field: Ident) -> P<ast::Expr> {
+ self.expr(span, ast::ExprKind::Field(expr, field))
+ }
+
pub fn expr_binary(
&self,
sp: Span,
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 8d4e36407..1d2b1298a 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -469,6 +469,7 @@ impl<'a> StripUnconfigured<'a> {
}
/// If attributes are not allowed on expressions, emit an error for `attr`
+ #[instrument(level = "trace", skip(self))]
pub(crate) fn maybe_emit_expr_attr_err(&self, attr: &Attribute) {
if !self.features.map_or(true, |features| features.stmt_expr_attributes) {
let mut err = feature_err(
@@ -486,9 +487,12 @@ impl<'a> StripUnconfigured<'a> {
}
}
- pub fn configure_expr(&self, expr: &mut P<ast::Expr>) {
- for attr in expr.attrs.iter() {
- self.maybe_emit_expr_attr_err(attr);
+ #[instrument(level = "trace", skip(self))]
+ pub fn configure_expr(&self, expr: &mut P<ast::Expr>, method_receiver: bool) {
+ if !method_receiver {
+ for attr in expr.attrs.iter() {
+ self.maybe_emit_expr_attr_err(attr);
+ }
}
// If an expr is valid to cfg away it will have been removed by the
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index 0feae0deb..d383f4832 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -1,46 +1,46 @@
-use rustc_macros::SessionDiagnostic;
+use rustc_macros::Diagnostic;
use rustc_span::symbol::MacroRulesNormalizedIdent;
use rustc_span::Span;
-#[derive(SessionDiagnostic)]
-#[diag(expand::expr_repeat_no_syntax_vars)]
+#[derive(Diagnostic)]
+#[diag(expand_expr_repeat_no_syntax_vars)]
pub(crate) struct NoSyntaxVarsExprRepeat {
#[primary_span]
pub span: Span,
}
-#[derive(SessionDiagnostic)]
-#[diag(expand::must_repeat_once)]
+#[derive(Diagnostic)]
+#[diag(expand_must_repeat_once)]
pub(crate) struct MustRepeatOnce {
#[primary_span]
pub span: Span,
}
-#[derive(SessionDiagnostic)]
-#[diag(expand::count_repetition_misplaced)]
+#[derive(Diagnostic)]
+#[diag(expand_count_repetition_misplaced)]
pub(crate) struct CountRepetitionMisplaced {
#[primary_span]
pub span: Span,
}
-#[derive(SessionDiagnostic)]
-#[diag(expand::meta_var_expr_unrecognized_var)]
+#[derive(Diagnostic)]
+#[diag(expand_meta_var_expr_unrecognized_var)]
pub(crate) struct MetaVarExprUnrecognizedVar {
#[primary_span]
pub span: Span,
pub key: MacroRulesNormalizedIdent,
}
-#[derive(SessionDiagnostic)]
-#[diag(expand::var_still_repeating)]
+#[derive(Diagnostic)]
+#[diag(expand_var_still_repeating)]
pub(crate) struct VarStillRepeating {
#[primary_span]
pub span: Span,
pub ident: MacroRulesNormalizedIdent,
}
-#[derive(SessionDiagnostic)]
-#[diag(expand::meta_var_dif_seq_matchers)]
+#[derive(Diagnostic)]
+#[diag(expand_meta_var_dif_seq_matchers)]
pub(crate) struct MetaVarsDifSeqMatchers {
#[primary_span]
pub span: Span,
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index c2add852a..57713fb3c 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -50,6 +50,7 @@ macro_rules! ast_fragments {
/// Can also serve as an input and intermediate result for macro expansion operations.
pub enum AstFragment {
OptExpr(Option<P<ast::Expr>>),
+ MethodReceiverExpr(P<ast::Expr>),
$($Kind($AstTy),)*
}
@@ -57,6 +58,7 @@ macro_rules! ast_fragments {
#[derive(Copy, Clone, PartialEq, Eq)]
pub enum AstFragmentKind {
OptExpr,
+ MethodReceiverExpr,
$($Kind,)*
}
@@ -64,6 +66,7 @@ macro_rules! ast_fragments {
pub fn name(self) -> &'static str {
match self {
AstFragmentKind::OptExpr => "expression",
+ AstFragmentKind::MethodReceiverExpr => "expression",
$(AstFragmentKind::$Kind => $kind_name,)*
}
}
@@ -72,6 +75,8 @@ macro_rules! ast_fragments {
match self {
AstFragmentKind::OptExpr =>
result.make_expr().map(Some).map(AstFragment::OptExpr),
+ AstFragmentKind::MethodReceiverExpr =>
+ result.make_expr().map(AstFragment::MethodReceiverExpr),
$(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
}
}
@@ -98,6 +103,13 @@ macro_rules! ast_fragments {
}
}
+ pub fn make_method_receiver_expr(self) -> P<ast::Expr> {
+ match self {
+ AstFragment::MethodReceiverExpr(expr) => expr,
+ _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+ }
+ }
+
$(pub fn $make_ast(self) -> $AstTy {
match self {
AstFragment::$Kind(ast) => ast,
@@ -120,6 +132,7 @@ macro_rules! ast_fragments {
}
});
}
+ AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
$($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
$($(AstFragment::$Kind(ast) =>
ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
@@ -130,6 +143,7 @@ macro_rules! ast_fragments {
match *self {
AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
AstFragment::OptExpr(None) => {}
+ AstFragment::MethodReceiverExpr(ref expr) => visitor.visit_method_receiver_expr(expr),
$($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)*
$($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] {
visitor.$visit_ast_elt(ast_elt, $($args)*);
@@ -222,6 +236,7 @@ impl AstFragmentKind {
match self {
AstFragmentKind::OptExpr
| AstFragmentKind::Expr
+ | AstFragmentKind::MethodReceiverExpr
| AstFragmentKind::Stmts
| AstFragmentKind::Ty
| AstFragmentKind::Pat => SupportsMacroExpansion::Yes { supports_inner_attrs: false },
@@ -285,6 +300,9 @@ impl AstFragmentKind {
AstFragmentKind::Expr => AstFragment::Expr(
items.next().expect("expected exactly one expression").expect_expr(),
),
+ AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(
+ items.next().expect("expected exactly one expression").expect_expr(),
+ ),
AstFragmentKind::OptExpr => {
AstFragment::OptExpr(items.next().map(Annotatable::expect_expr))
}
@@ -327,7 +345,7 @@ impl InvocationKind {
fn placeholder_visibility(&self) -> Option<ast::Visibility> {
// HACK: For unnamed fields placeholders should have the same visibility as the actual
// fields because for tuple structs/variants resolve determines visibilities of their
- // constructor using these field visibilities before attributes on them are are expanded.
+ // constructor using these field visibilities before attributes on them are expanded.
// The assumption is that the attribute expansion cannot change field visibilities,
// and it holds because only inert attributes are supported in this position.
match self {
@@ -893,6 +911,7 @@ pub fn parse_ast_fragment<'a>(
AstFragment::Stmts(stmts)
}
AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?),
+ AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(this.parse_expr()?),
AstFragmentKind::OptExpr => {
if this.token != token::Eof {
AstFragment::OptExpr(Some(this.parse_expr()?))
@@ -937,13 +956,12 @@ pub fn ensure_complete_parse<'a>(
kind_name,
);
err.note(&msg);
- let semi_span = this.sess.source_map().next_point(span);
- let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span));
- match this.sess.source_map().span_to_snippet(semi_full_span) {
+ let semi_span = this.sess.source_map().next_point(span);
+ match this.sess.source_map().span_to_snippet(semi_span) {
Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
err.span_suggestion(
- semi_span,
+ span.shrink_to_hi(),
"you might be missing a semicolon here",
";",
Applicability::MaybeIncorrect,
@@ -1478,6 +1496,42 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> {
}
}
+/// This struct is a hack to workaround unstable of `stmt_expr_attributes`.
+/// It can be removed once that feature is stabilized.
+struct MethodReceiverTag;
+impl DummyAstNode for MethodReceiverTag {
+ fn dummy() -> MethodReceiverTag {
+ MethodReceiverTag
+ }
+}
+impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
+ type OutputTy = Self;
+ type AttrsTy = ast::AttrVec;
+ const KIND: AstFragmentKind = AstFragmentKind::MethodReceiverExpr;
+ fn descr() -> &'static str {
+ "an expression"
+ }
+ fn to_annotatable(self) -> Annotatable {
+ Annotatable::Expr(self.wrapped)
+ }
+ fn fragment_to_output(fragment: AstFragment) -> Self::OutputTy {
+ AstNodeWrapper::new(fragment.make_method_receiver_expr(), MethodReceiverTag)
+ }
+ fn noop_visit<V: MutVisitor>(&mut self, visitor: &mut V) {
+ noop_visit_expr(&mut self.wrapped, visitor)
+ }
+ fn is_mac_call(&self) -> bool {
+ matches!(self.wrapped.kind, ast::ExprKind::MacCall(..))
+ }
+ fn take_mac_call(self) -> (P<ast::MacCall>, Self::AttrsTy, AddSemicolon) {
+ let node = self.wrapped.into_inner();
+ match node.kind {
+ ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
+ _ => unreachable!(),
+ }
+ }
+}
+
struct InvocationCollector<'a, 'b> {
cx: &'a mut ExtCtxt<'b>,
invocations: Vec<(Invocation, Option<Lrc<SyntaxExtension>>)>,
@@ -1841,6 +1895,14 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
self.visit_node(node)
}
+ fn visit_method_receiver_expr(&mut self, node: &mut P<ast::Expr>) {
+ visit_clobber(node, |node| {
+ let mut wrapper = AstNodeWrapper::new(node, MethodReceiverTag);
+ self.visit_node(&mut wrapper);
+ wrapper.wrapped
+ })
+ }
+
fn filter_map_expr(&mut self, node: P<ast::Expr>) -> Option<P<ast::Expr>> {
self.flat_map_node(AstNodeWrapper::new(node, OptExprTag))
}
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index ffc9abe64..b34de94fb 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -3,7 +3,6 @@
#![feature(associated_type_defaults)]
#![feature(if_let_guard)]
#![feature(let_chains)]
-#![cfg_attr(bootstrap, feature(let_else))]
#![feature(macro_metavar_expr)]
#![feature(proc_macro_diagnostic)]
#![feature(proc_macro_internals)]
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 7764ffd24..f6fe38174 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -14,7 +14,7 @@ use rustc_ast::{NodeId, DUMMY_NODE_ID};
use rustc_ast_pretty::pprust;
use rustc_attr::{self as attr, TransparencyError};
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
-use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder};
+use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage};
use rustc_feature::Features;
use rustc_lint_defs::builtin::{
RUST_2021_INCOMPATIBLE_OR_PATTERNS, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
@@ -68,19 +68,22 @@ fn emit_frag_parse_err(
kind: AstFragmentKind,
) {
// FIXME(davidtwco): avoid depending on the error message text
- if parser.token == token::Eof && e.message[0].0.expect_str().ends_with(", found `<eof>`") {
- if !e.span.is_dummy() {
- // early end of macro arm (#52866)
- e.replace_span_with(parser.sess.source_map().next_point(parser.token.span));
- }
+ if parser.token == token::Eof
+ && let DiagnosticMessage::Str(message) = &e.message[0].0
+ && message.ends_with(", found `<eof>`")
+ {
let msg = &e.message[0];
e.message[0] = (
- rustc_errors::DiagnosticMessage::Str(format!(
+ DiagnosticMessage::Str(format!(
"macro expansion ends with an incomplete expression: {}",
- msg.0.expect_str().replace(", found `<eof>`", ""),
+ message.replace(", found `<eof>`", ""),
)),
msg.1,
);
+ if !e.span.is_dummy() {
+ // early end of macro arm (#52866)
+ e.replace_span_with(parser.token.span.shrink_to_hi());
+ }
}
if e.span.is_dummy() {
// Get around lack of span in error (#30128)
@@ -247,6 +250,7 @@ fn expand_macro<'cx>(
// hacky, but speeds up the `html5ever` benchmark significantly. (Issue
// 68836 suggests a more comprehensive but more complex change to deal with
// this situation.)
+ // FIXME(Nilstrieb): Stop recovery from happening on this parser and retry later with recovery if the macro failed to match.
let parser = parser_from_cx(sess, arg.clone());
// Try each arm's matchers.
@@ -593,14 +597,14 @@ pub fn compile_declarative_macro(
(mk_syn_ext(expander), rule_spans)
}
-#[derive(SessionSubdiagnostic)]
+#[derive(Subdiagnostic)]
enum ExplainDocComment {
- #[label(expand::explain_doc_comment_inner)]
+ #[label(expand_explain_doc_comment_inner)]
Inner {
#[primary_span]
span: Span,
},
- #[label(expand::explain_doc_comment_outer)]
+ #[label(expand_explain_doc_comment_outer)]
Outer {
#[primary_span]
span: Span,
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 3b0d5ddb9..faaf3b3fe 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -55,6 +55,7 @@ pub fn placeholder(
}),
AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()),
AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())),
+ AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(expr_placeholder()),
AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item {
id,
span,
@@ -296,6 +297,13 @@ impl MutVisitor for PlaceholderExpander {
}
}
+ fn visit_method_receiver_expr(&mut self, expr: &mut P<ast::Expr>) {
+ match expr.kind {
+ ast::ExprKind::MacCall(_) => *expr = self.remove(expr.id).make_method_receiver_expr(),
+ _ => noop_visit_expr(expr, self),
+ }
+ }
+
fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
match expr.kind {
ast::ExprKind::MacCall(_) => self.remove(expr.id).make_opt_expr(),
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 59a7b668a..cc2858d3f 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -1,5 +1,8 @@
use crate::base::ExtCtxt;
-
+use pm::bridge::{
+ server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
+};
+use pm::{Delimiter, Level, LineColumn};
use rustc_ast as ast;
use rustc_ast::token;
use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
@@ -13,11 +16,7 @@ use rustc_session::parse::ParseSess;
use rustc_span::def_id::CrateNum;
use rustc_span::symbol::{self, sym, Symbol};
use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
-
-use pm::bridge::{
- server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
-};
-use pm::{Delimiter, Level, LineColumn};
+use smallvec::{smallvec, SmallVec};
use std::ops::Bound;
trait FromInternal<T> {
@@ -110,10 +109,26 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
};
+ // Split the operator into one or more `Punct`s, one per character.
+ // The final one inherits the jointness of the original token. Any
+ // before that get `joint = true`.
let mut op = |s: &str| {
assert!(s.is_ascii());
- trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| {
- TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span })
+ trees.extend(s.bytes().enumerate().map(|(i, ch)| {
+ let is_final = i == s.len() - 1;
+ // Split the token span into single chars. Unless the span
+ // is an unusual one, e.g. due to proc macro expansion. We
+ // determine this by assuming any span with a length that
+ // matches the operator length is a normal one, and any
+ // span with a different length is an unusual one.
+ let span = if (span.hi() - span.lo()).to_usize() == s.len() {
+ let lo = span.lo() + BytePos::from_usize(i);
+ let hi = lo + BytePos::from_usize(1);
+ span.with_lo(lo).with_hi(hi)
+ } else {
+ span
+ };
+ TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
}));
};
@@ -237,23 +252,57 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
}
}
-impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>) {
- fn to_internal(self) -> TokenStream {
+// We use a `SmallVec` because the output size is always one or two `TokenTree`s.
+impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
+ for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>)
+{
+ fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> {
use rustc_ast::token::*;
let (tree, rustc) = self;
- let (ch, joint, span) = match tree {
- TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
+ match tree {
+ TokenTree::Punct(Punct { ch, joint, span }) => {
+ let kind = match ch {
+ b'=' => Eq,
+ b'<' => Lt,
+ b'>' => Gt,
+ b'!' => Not,
+ b'~' => Tilde,
+ b'+' => BinOp(Plus),
+ b'-' => BinOp(Minus),
+ b'*' => BinOp(Star),
+ b'/' => BinOp(Slash),
+ b'%' => BinOp(Percent),
+ b'^' => BinOp(Caret),
+ b'&' => BinOp(And),
+ b'|' => BinOp(Or),
+ b'@' => At,
+ b'.' => Dot,
+ b',' => Comma,
+ b';' => Semi,
+ b':' => Colon,
+ b'#' => Pound,
+ b'$' => Dollar,
+ b'?' => Question,
+ b'\'' => SingleQuote,
+ _ => unreachable!(),
+ };
+ smallvec![if joint {
+ tokenstream::TokenTree::token_joint(kind, span)
+ } else {
+ tokenstream::TokenTree::token_alone(kind, span)
+ }]
+ }
TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
- return tokenstream::TokenStream::delimited(
+ smallvec![tokenstream::TokenTree::Delimited(
tokenstream::DelimSpan { open, close },
delimiter.to_internal(),
stream.unwrap_or_default(),
- );
+ )]
}
TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
rustc.sess().symbol_gallery.insert(sym, span);
- return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span);
+ smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)]
}
TokenTree::Literal(self::Literal {
kind: self::LitKind::Integer,
@@ -266,7 +315,7 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
- return [a, b].into_iter().collect();
+ smallvec![a, b]
}
TokenTree::Literal(self::Literal {
kind: self::LitKind::Float,
@@ -279,46 +328,14 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
- return [a, b].into_iter().collect();
+ smallvec![a, b]
}
TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
- return tokenstream::TokenStream::token_alone(
+ smallvec![tokenstream::TokenTree::token_alone(
TokenKind::lit(kind.to_internal(), symbol, suffix),
span,
- );
+ )]
}
- };
-
- let kind = match ch {
- b'=' => Eq,
- b'<' => Lt,
- b'>' => Gt,
- b'!' => Not,
- b'~' => Tilde,
- b'+' => BinOp(Plus),
- b'-' => BinOp(Minus),
- b'*' => BinOp(Star),
- b'/' => BinOp(Slash),
- b'%' => BinOp(Percent),
- b'^' => BinOp(Caret),
- b'&' => BinOp(And),
- b'|' => BinOp(Or),
- b'@' => At,
- b'.' => Dot,
- b',' => Comma,
- b';' => Semi,
- b':' => Colon,
- b'#' => Pound,
- b'$' => Dollar,
- b'?' => Question,
- b'\'' => SingleQuote,
- _ => unreachable!(),
- };
-
- if joint {
- tokenstream::TokenStream::token_joint(kind, span)
- } else {
- tokenstream::TokenStream::token_alone(kind, span)
}
}
}
@@ -533,7 +550,7 @@ impl server::TokenStream for Rustc<'_, '_> {
&mut self,
tree: TokenTree<Self::TokenStream, Self::Span, Self::Symbol>,
) -> Self::TokenStream {
- (tree, &mut *self).to_internal()
+ Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::<Vec<_>>())
}
fn concat_trees(
@@ -541,14 +558,14 @@ impl server::TokenStream for Rustc<'_, '_> {
base: Option<Self::TokenStream>,
trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>>,
) -> Self::TokenStream {
- let mut builder = tokenstream::TokenStreamBuilder::new();
- if let Some(base) = base {
- builder.push(base);
- }
+ let mut stream =
+ if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
for tree in trees {
- builder.push((tree, &mut *self).to_internal());
+ for tt in (tree, &mut *self).to_internal() {
+ stream.push_tree(tt);
+ }
}
- builder.build()
+ stream
}
fn concat_streams(
@@ -556,14 +573,12 @@ impl server::TokenStream for Rustc<'_, '_> {
base: Option<Self::TokenStream>,
streams: Vec<Self::TokenStream>,
) -> Self::TokenStream {
- let mut builder = tokenstream::TokenStreamBuilder::new();
- if let Some(base) = base {
- builder.push(base);
+ let mut stream =
+ if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
+ for s in streams {
+ stream.push_stream(s);
}
- for stream in streams {
- builder.push(stream);
- }
- builder.build()
+ stream
}
fn into_trees(
@@ -689,6 +704,7 @@ impl server::Span for Rustc<'_, '_> {
fn source_text(&mut self, span: Self::Span) -> Option<String> {
self.sess().source_map().span_to_snippet(span).ok()
}
+
/// Saves the provided span into the metadata of
/// *the crate we are currently compiling*, which must
/// be a proc-macro crate. This id can be passed to
diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs
index eed696810..91c4dd732 100644
--- a/compiler/rustc_expand/src/tokenstream/tests.rs
+++ b/compiler/rustc_expand/src/tokenstream/tests.rs
@@ -1,7 +1,7 @@
use crate::tests::string_to_stream;
use rustc_ast::token;
-use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
+use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_span::create_default_session_globals_then;
use rustc_span::{BytePos, Span, Symbol};
@@ -19,10 +19,9 @@ fn test_concat() {
let test_res = string_to_ts("foo::bar::baz");
let test_fst = string_to_ts("foo::bar");
let test_snd = string_to_ts("::baz");
- let mut builder = TokenStreamBuilder::new();
- builder.push(test_fst);
- builder.push(test_snd);
- let eq_res = builder.build();
+ let mut eq_res = TokenStream::default();
+ eq_res.push_stream(test_fst);
+ eq_res.push_stream(test_snd);
assert_eq!(test_res.trees().count(), 5);
assert_eq!(eq_res.trees().count(), 5);
assert_eq!(test_res.eq_unspanned(&eq_res), true);
@@ -99,11 +98,10 @@ fn test_is_empty() {
#[test]
fn test_dotdotdot() {
create_default_session_globals_then(|| {
- let mut builder = TokenStreamBuilder::new();
- builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
- builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
- builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
- let stream = builder.build();
+ let mut stream = TokenStream::default();
+ stream.push_tree(TokenTree::token_joint(token::Dot, sp(0, 1)));
+ stream.push_tree(TokenTree::token_joint(token::Dot, sp(1, 2)));
+ stream.push_tree(TokenTree::token_alone(token::Dot, sp(2, 3)));
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);
})