From 698f8c2f01ea549d77d7dc3338a12e04c11057b9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:02:58 +0200 Subject: Adding upstream version 1.64.0+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_const_eval/src/const_eval/error.rs | 252 ++++++++++ .../src/const_eval/eval_queries.rs | 395 +++++++++++++++ .../rustc_const_eval/src/const_eval/fn_queries.rs | 82 ++++ .../rustc_const_eval/src/const_eval/machine.rs | 527 +++++++++++++++++++++ compiler/rustc_const_eval/src/const_eval/mod.rs | 163 +++++++ .../rustc_const_eval/src/const_eval/valtrees.rs | 475 +++++++++++++++++++ 6 files changed, 1894 insertions(+) create mode 100644 compiler/rustc_const_eval/src/const_eval/error.rs create mode 100644 compiler/rustc_const_eval/src/const_eval/eval_queries.rs create mode 100644 compiler/rustc_const_eval/src/const_eval/fn_queries.rs create mode 100644 compiler/rustc_const_eval/src/const_eval/machine.rs create mode 100644 compiler/rustc_const_eval/src/const_eval/mod.rs create mode 100644 compiler/rustc_const_eval/src/const_eval/valtrees.rs (limited to 'compiler/rustc_const_eval/src/const_eval') diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs new file mode 100644 index 000000000..322bfd5ce --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/error.rs @@ -0,0 +1,252 @@ +use std::error::Error; +use std::fmt; + +use rustc_errors::Diagnostic; +use rustc_hir as hir; +use rustc_middle::mir::AssertKind; +use rustc_middle::ty::{layout::LayoutError, query::TyCtxtAt, ConstInt}; +use rustc_span::{Span, Symbol}; + +use super::InterpCx; +use crate::interpret::{ + struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType, UnsupportedOpInfo, +}; + +/// The CTFE machine has some custom error kinds. +#[derive(Clone, Debug)] +pub enum ConstEvalErrKind { + NeedsRfc(String), + ConstAccessesStatic, + ModifiedGlobal, + AssertFailure(AssertKind), + Panic { msg: Symbol, line: u32, col: u32, file: Symbol }, + Abort(String), +} + +impl MachineStopType for ConstEvalErrKind { + fn is_hard_err(&self) -> bool { + matches!(self, Self::Panic { .. }) + } +} + +// The errors become `MachineStop` with plain strings when being raised. +// `ConstEvalErr` (in `librustc_middle/mir/interpret/error.rs`) knows to +// handle these. +impl<'tcx> Into> for ConstEvalErrKind { + fn into(self) -> InterpErrorInfo<'tcx> { + err_machine_stop!(self).into() + } +} + +impl fmt::Display for ConstEvalErrKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use self::ConstEvalErrKind::*; + match *self { + NeedsRfc(ref msg) => { + write!(f, "\"{}\" needs an rfc before being allowed inside constants", msg) + } + ConstAccessesStatic => write!(f, "constant accesses static"), + ModifiedGlobal => { + write!(f, "modifying a static's initial value from another static's initializer") + } + AssertFailure(ref msg) => write!(f, "{:?}", msg), + Panic { msg, line, col, file } => { + write!(f, "the evaluated program panicked at '{}', {}:{}:{}", msg, file, line, col) + } + Abort(ref msg) => write!(f, "{}", msg), + } + } +} + +impl Error for ConstEvalErrKind {} + +/// When const-evaluation errors, this type is constructed with the resulting information, +/// and then used to emit the error as a lint or hard error. +#[derive(Debug)] +pub struct ConstEvalErr<'tcx> { + pub span: Span, + pub error: InterpError<'tcx>, + pub stacktrace: Vec>, +} + +impl<'tcx> ConstEvalErr<'tcx> { + /// Turn an interpreter error into something to report to the user. + /// As a side-effect, if RUSTC_CTFE_BACKTRACE is set, this prints the backtrace. + /// Should be called only if the error is actually going to to be reported! + pub fn new<'mir, M: Machine<'mir, 'tcx>>( + ecx: &InterpCx<'mir, 'tcx, M>, + error: InterpErrorInfo<'tcx>, + span: Option, + ) -> ConstEvalErr<'tcx> + where + 'tcx: 'mir, + { + error.print_backtrace(); + let mut stacktrace = ecx.generate_stacktrace(); + // Filter out `requires_caller_location` frames. + stacktrace.retain(|frame| !frame.instance.def.requires_caller_location(*ecx.tcx)); + // If `span` is missing, use topmost remaining frame, or else the "root" span from `ecx.tcx`. + let span = span.or_else(|| stacktrace.first().map(|f| f.span)).unwrap_or(ecx.tcx.span); + ConstEvalErr { error: error.into_kind(), stacktrace, span } + } + + pub fn struct_error( + &self, + tcx: TyCtxtAt<'tcx>, + message: &str, + decorate: impl FnOnce(&mut Diagnostic), + ) -> ErrorHandled { + self.struct_generic(tcx, message, decorate, None) + } + + pub fn report_as_error(&self, tcx: TyCtxtAt<'tcx>, message: &str) -> ErrorHandled { + self.struct_error(tcx, message, |_| {}) + } + + pub fn report_as_lint( + &self, + tcx: TyCtxtAt<'tcx>, + message: &str, + lint_root: hir::HirId, + span: Option, + ) -> ErrorHandled { + self.struct_generic( + tcx, + message, + |lint: &mut Diagnostic| { + // Apply the span. + if let Some(span) = span { + let primary_spans = lint.span.primary_spans().to_vec(); + // point at the actual error as the primary span + lint.replace_span_with(span); + // point to the `const` statement as a secondary span + // they don't have any label + for sp in primary_spans { + if sp != span { + lint.span_label(sp, ""); + } + } + } + }, + Some(lint_root), + ) + } + + /// Create a diagnostic for this const eval error. + /// + /// Sets the message passed in via `message` and adds span labels with detailed error + /// information before handing control back to `decorate` to do any final annotations, + /// after which the diagnostic is emitted. + /// + /// If `lint_root.is_some()` report it as a lint, else report it as a hard error. + /// (Except that for some errors, we ignore all that -- see `must_error` below.) + #[instrument(skip(self, tcx, decorate, lint_root), level = "debug")] + fn struct_generic( + &self, + tcx: TyCtxtAt<'tcx>, + message: &str, + decorate: impl FnOnce(&mut Diagnostic), + lint_root: Option, + ) -> ErrorHandled { + let finish = |err: &mut Diagnostic, span_msg: Option| { + trace!("reporting const eval failure at {:?}", self.span); + if let Some(span_msg) = span_msg { + err.span_label(self.span, span_msg); + } + // Add some more context for select error types. + match self.error { + InterpError::Unsupported( + UnsupportedOpInfo::ReadPointerAsBytes + | UnsupportedOpInfo::PartialPointerOverwrite(_) + ) => { + err.help("this code performed an operation that depends on the underlying bytes representing a pointer"); + err.help("the absolute address of a pointer is not known at compile-time, so such operations are not supported"); + } + _ => {} + } + // Add spans for the stacktrace. Don't print a single-line backtrace though. + if self.stacktrace.len() > 1 { + // Helper closure to print duplicated lines. + let mut flush_last_line = |last_frame, times| { + if let Some((line, span)) = last_frame { + err.span_label(span, &line); + // Don't print [... additional calls ...] if the number of lines is small + if times < 3 { + for _ in 0..times { + err.span_label(span, &line); + } + } else { + err.span_label( + span, + format!("[... {} additional calls {} ...]", times, &line), + ); + } + } + }; + + let mut last_frame = None; + let mut times = 0; + for frame_info in &self.stacktrace { + let frame = (frame_info.to_string(), frame_info.span); + if last_frame.as_ref() == Some(&frame) { + times += 1; + } else { + flush_last_line(last_frame, times); + last_frame = Some(frame); + times = 0; + } + } + flush_last_line(last_frame, times); + } + // Let the caller attach any additional information it wants. + decorate(err); + }; + + debug!("self.error: {:?}", self.error); + // Special handling for certain errors + match &self.error { + // Don't emit a new diagnostic for these errors + err_inval!(Layout(LayoutError::Unknown(_))) | err_inval!(TooGeneric) => { + return ErrorHandled::TooGeneric; + } + err_inval!(AlreadyReported(error_reported)) => { + return ErrorHandled::Reported(*error_reported); + } + err_inval!(Layout(LayoutError::SizeOverflow(_))) => { + // We must *always* hard error on these, even if the caller wants just a lint. + // The `message` makes little sense here, this is a more serious error than the + // caller thinks anyway. + // See . + let mut err = struct_error(tcx, &self.error.to_string()); + finish(&mut err, None); + return ErrorHandled::Reported(err.emit()); + } + _ => {} + }; + + let err_msg = self.error.to_string(); + + // Regular case - emit a lint. + if let Some(lint_root) = lint_root { + // Report as lint. + let hir_id = + self.stacktrace.iter().rev().find_map(|frame| frame.lint_root).unwrap_or(lint_root); + tcx.struct_span_lint_hir( + rustc_session::lint::builtin::CONST_ERR, + hir_id, + tcx.span, + |lint| { + let mut lint = lint.build(message); + finish(&mut lint, Some(err_msg)); + lint.emit(); + }, + ); + ErrorHandled::Linted + } else { + // Report as hard error. + let mut err = struct_error(tcx, message); + finish(&mut err, Some(err_msg)); + ErrorHandled::Reported(err.emit()) + } + } +} diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs new file mode 100644 index 000000000..975fb4b22 --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -0,0 +1,395 @@ +use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr}; +use crate::interpret::eval_nullary_intrinsic; +use crate::interpret::{ + intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, + Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, + ScalarMaybeUninit, StackPopCleanup, InterpError, +}; + +use rustc_hir::def::DefKind; +use rustc_middle::mir; +use rustc_middle::mir::interpret::ErrorHandled; +use rustc_middle::mir::pretty::display_allocation; +use rustc_middle::traits::Reveal; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::{self, subst::Subst, TyCtxt}; +use rustc_span::source_map::Span; +use rustc_target::abi::{self, Abi}; +use std::borrow::Cow; +use std::convert::TryInto; + +const NOTE_ON_UNDEFINED_BEHAVIOR_ERROR: &str = "The rules on what exactly is undefined behavior aren't clear, \ + so this check might be overzealous. Please open an issue on the rustc \ + repository if you believe it should not be considered undefined behavior."; + +// Returns a pointer to where the result lives +fn eval_body_using_ecx<'mir, 'tcx>( + ecx: &mut CompileTimeEvalContext<'mir, 'tcx>, + cid: GlobalId<'tcx>, + body: &'mir mir::Body<'tcx>, +) -> InterpResult<'tcx, MPlaceTy<'tcx>> { + debug!("eval_body_using_ecx: {:?}, {:?}", cid, ecx.param_env); + let tcx = *ecx.tcx; + assert!( + cid.promoted.is_some() + || matches!( + ecx.tcx.def_kind(cid.instance.def_id()), + DefKind::Const + | DefKind::Static(_) + | DefKind::ConstParam + | DefKind::AnonConst + | DefKind::InlineConst + | DefKind::AssocConst + ), + "Unexpected DefKind: {:?}", + ecx.tcx.def_kind(cid.instance.def_id()) + ); + let layout = ecx.layout_of(body.bound_return_ty().subst(tcx, cid.instance.substs))?; + assert!(!layout.is_unsized()); + let ret = ecx.allocate(layout, MemoryKind::Stack)?; + + trace!( + "eval_body_using_ecx: pushing stack frame for global: {}{}", + with_no_trimmed_paths!(ty::tls::with(|tcx| tcx.def_path_str(cid.instance.def_id()))), + cid.promoted.map_or_else(String::new, |p| format!("::promoted[{:?}]", p)) + ); + + ecx.push_stack_frame( + cid.instance, + body, + &ret.into(), + StackPopCleanup::Root { cleanup: false }, + )?; + + // The main interpreter loop. + ecx.run()?; + + // Intern the result + let intern_kind = if cid.promoted.is_some() { + InternKind::Promoted + } else { + match tcx.static_mutability(cid.instance.def_id()) { + Some(m) => InternKind::Static(m), + None => InternKind::Constant, + } + }; + intern_const_alloc_recursive(ecx, intern_kind, &ret)?; + + debug!("eval_body_using_ecx done: {:?}", *ret); + Ok(ret) +} + +/// The `InterpCx` is only meant to be used to do field and index projections into constants for +/// `simd_shuffle` and const patterns in match arms. +/// +/// The function containing the `match` that is currently being analyzed may have generic bounds +/// that inform us about the generic bounds of the constant. E.g., using an associated constant +/// of a function's generic parameter will require knowledge about the bounds on the generic +/// parameter. These bounds are passed to `mk_eval_cx` via the `ParamEnv` argument. +pub(super) fn mk_eval_cx<'mir, 'tcx>( + tcx: TyCtxt<'tcx>, + root_span: Span, + param_env: ty::ParamEnv<'tcx>, + can_access_statics: bool, +) -> CompileTimeEvalContext<'mir, 'tcx> { + debug!("mk_eval_cx: {:?}", param_env); + InterpCx::new( + tcx, + root_span, + param_env, + CompileTimeInterpreter::new(tcx.const_eval_limit(), can_access_statics), + ) +} + +/// This function converts an interpreter value into a constant that is meant for use in the +/// type system. +#[instrument(skip(ecx), level = "debug")] +pub(super) fn op_to_const<'tcx>( + ecx: &CompileTimeEvalContext<'_, 'tcx>, + op: &OpTy<'tcx>, +) -> ConstValue<'tcx> { + // We do not have value optimizations for everything. + // Only scalars and slices, since they are very common. + // Note that further down we turn scalars of uninitialized bits back to `ByRef`. These can result + // from scalar unions that are initialized with one of their zero sized variants. We could + // instead allow `ConstValue::Scalar` to store `ScalarMaybeUninit`, but that would affect all + // the usual cases of extracting e.g. a `usize`, without there being a real use case for the + // `Undef` situation. + let try_as_immediate = match op.layout.abi { + Abi::Scalar(abi::Scalar::Initialized { .. }) => true, + Abi::ScalarPair(..) => match op.layout.ty.kind() { + ty::Ref(_, inner, _) => match *inner.kind() { + ty::Slice(elem) => elem == ecx.tcx.types.u8, + ty::Str => true, + _ => false, + }, + _ => false, + }, + _ => false, + }; + let immediate = if try_as_immediate { + Err(ecx.read_immediate(op).expect("normalization works on validated constants")) + } else { + // It is guaranteed that any non-slice scalar pair is actually ByRef here. + // When we come back from raw const eval, we are always by-ref. The only way our op here is + // by-val is if we are in destructure_mir_constant, i.e., if this is (a field of) something that we + // "tried to make immediate" before. We wouldn't do that for non-slice scalar pairs or + // structs containing such. + op.try_as_mplace() + }; + + debug!(?immediate); + + // We know `offset` is relative to the allocation, so we can use `into_parts`. + let to_const_value = |mplace: &MPlaceTy<'_>| { + debug!("to_const_value(mplace: {:?})", mplace); + match mplace.ptr.into_parts() { + (Some(alloc_id), offset) => { + let alloc = ecx.tcx.global_alloc(alloc_id).unwrap_memory(); + ConstValue::ByRef { alloc, offset } + } + (None, offset) => { + assert!(mplace.layout.is_zst()); + assert_eq!( + offset.bytes() % mplace.layout.align.abi.bytes(), + 0, + "this MPlaceTy must come from a validated constant, thus we can assume the \ + alignment is correct", + ); + ConstValue::ZeroSized + } + } + }; + match immediate { + Ok(ref mplace) => to_const_value(mplace), + // see comment on `let try_as_immediate` above + Err(imm) => match *imm { + _ if imm.layout.is_zst() => ConstValue::ZeroSized, + Immediate::Scalar(x) => match x { + ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s), + ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place()), + }, + Immediate::ScalarPair(a, b) => { + debug!("ScalarPair(a: {:?}, b: {:?})", a, b); + // We know `offset` is relative to the allocation, so we can use `into_parts`. + let (data, start) = match a.to_pointer(ecx).unwrap().into_parts() { + (Some(alloc_id), offset) => { + (ecx.tcx.global_alloc(alloc_id).unwrap_memory(), offset.bytes()) + } + (None, _offset) => ( + ecx.tcx.intern_const_alloc(Allocation::from_bytes_byte_aligned_immutable( + b"" as &[u8], + )), + 0, + ), + }; + let len = b.to_machine_usize(ecx).unwrap(); + let start = start.try_into().unwrap(); + let len: usize = len.try_into().unwrap(); + ConstValue::Slice { data, start, end: start + len } + } + Immediate::Uninit => to_const_value(&op.assert_mem_place()), + }, + } +} + +#[instrument(skip(tcx), level = "debug")] +pub(crate) fn turn_into_const_value<'tcx>( + tcx: TyCtxt<'tcx>, + constant: ConstAlloc<'tcx>, + key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>, +) -> ConstValue<'tcx> { + let cid = key.value; + let def_id = cid.instance.def.def_id(); + let is_static = tcx.is_static(def_id); + let ecx = mk_eval_cx(tcx, tcx.def_span(key.value.instance.def_id()), key.param_env, is_static); + + let mplace = ecx.raw_const_to_mplace(constant).expect( + "can only fail if layout computation failed, \ + which should have given a good error before ever invoking this function", + ); + assert!( + !is_static || cid.promoted.is_some(), + "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead" + ); + + // Turn this into a proper constant. + let const_val = op_to_const(&ecx, &mplace.into()); + debug!(?const_val); + + const_val +} + +#[instrument(skip(tcx), level = "debug")] +pub fn eval_to_const_value_raw_provider<'tcx>( + tcx: TyCtxt<'tcx>, + key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>, +) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> { + assert!(key.param_env.is_const()); + // see comment in eval_to_allocation_raw_provider for what we're doing here + if key.param_env.reveal() == Reveal::All { + let mut key = key; + key.param_env = key.param_env.with_user_facing(); + match tcx.eval_to_const_value_raw(key) { + // try again with reveal all as requested + Err(ErrorHandled::TooGeneric) => {} + // deduplicate calls + other => return other, + } + } + + // We call `const_eval` for zero arg intrinsics, too, in order to cache their value. + // Catch such calls and evaluate them instead of trying to load a constant's MIR. + if let ty::InstanceDef::Intrinsic(def_id) = key.value.instance.def { + let ty = key.value.instance.ty(tcx, key.param_env); + let ty::FnDef(_, substs) = ty.kind() else { + bug!("intrinsic with type {:?}", ty); + }; + return eval_nullary_intrinsic(tcx, key.param_env, def_id, substs).map_err(|error| { + let span = tcx.def_span(def_id); + let error = ConstEvalErr { error: error.into_kind(), stacktrace: vec![], span }; + error.report_as_error(tcx.at(span), "could not evaluate nullary intrinsic") + }); + } + + tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key)) +} + +#[instrument(skip(tcx), level = "debug")] +pub fn eval_to_allocation_raw_provider<'tcx>( + tcx: TyCtxt<'tcx>, + key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>, +) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> { + assert!(key.param_env.is_const()); + // Because the constant is computed twice (once per value of `Reveal`), we are at risk of + // reporting the same error twice here. To resolve this, we check whether we can evaluate the + // constant in the more restrictive `Reveal::UserFacing`, which most likely already was + // computed. For a large percentage of constants that will already have succeeded. Only + // associated constants of generic functions will fail due to not enough monomorphization + // information being available. + + // In case we fail in the `UserFacing` variant, we just do the real computation. + if key.param_env.reveal() == Reveal::All { + let mut key = key; + key.param_env = key.param_env.with_user_facing(); + match tcx.eval_to_allocation_raw(key) { + // try again with reveal all as requested + Err(ErrorHandled::TooGeneric) => {} + // deduplicate calls + other => return other, + } + } + if cfg!(debug_assertions) { + // Make sure we format the instance even if we do not print it. + // This serves as a regression test against an ICE on printing. + // The next two lines concatenated contain some discussion: + // https://rust-lang.zulipchat.com/#narrow/stream/146212-t-compiler.2Fconst-eval/ + // subject/anon_const_instance_printing/near/135980032 + let instance = with_no_trimmed_paths!(key.value.instance.to_string()); + trace!("const eval: {:?} ({})", key, instance); + } + + let cid = key.value; + let def = cid.instance.def.with_opt_param(); + let is_static = tcx.is_static(def.did); + + let mut ecx = InterpCx::new( + tcx, + tcx.def_span(def.did), + key.param_env, + // Statics (and promoteds inside statics) may access other statics, because unlike consts + // they do not have to behave "as if" they were evaluated at runtime. + CompileTimeInterpreter::new(tcx.const_eval_limit(), /*can_access_statics:*/ is_static), + ); + + let res = ecx.load_mir(cid.instance.def, cid.promoted); + match res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, &body)) { + Err(error) => { + let err = ConstEvalErr::new(&ecx, error, None); + // Some CTFE errors raise just a lint, not a hard error; see + // . + let is_hard_err = if let Some(def) = def.as_local() { + // (Associated) consts only emit a lint, since they might be unused. + !matches!(tcx.def_kind(def.did.to_def_id()), DefKind::Const | DefKind::AssocConst) + // check if the inner InterpError is hard + || err.error.is_hard_err() + } else { + // use of broken constant from other crate: always an error + true + }; + + if is_hard_err { + let msg = if is_static { + Cow::from("could not evaluate static initializer") + } else { + // If the current item has generics, we'd like to enrich the message with the + // instance and its substs: to show the actual compile-time values, in addition to + // the expression, leading to the const eval error. + let instance = &key.value.instance; + if !instance.substs.is_empty() { + let instance = with_no_trimmed_paths!(instance.to_string()); + let msg = format!("evaluation of `{}` failed", instance); + Cow::from(msg) + } else { + Cow::from("evaluation of constant value failed") + } + }; + + Err(err.report_as_error(ecx.tcx.at(err.span), &msg)) + } else { + let hir_id = tcx.hir().local_def_id_to_hir_id(def.as_local().unwrap().did); + Err(err.report_as_lint( + tcx.at(tcx.def_span(def.did)), + "any use of this value will cause an error", + hir_id, + Some(err.span), + )) + } + } + Ok(mplace) => { + // Since evaluation had no errors, validate the resulting constant. + // This is a separate `try` block to provide more targeted error reporting. + let validation = try { + let mut ref_tracking = RefTracking::new(mplace); + let mut inner = false; + while let Some((mplace, path)) = ref_tracking.todo.pop() { + let mode = match tcx.static_mutability(cid.instance.def_id()) { + Some(_) if cid.promoted.is_some() => { + // Promoteds in statics are allowed to point to statics. + CtfeValidationMode::Const { inner, allow_static_ptrs: true } + } + Some(_) => CtfeValidationMode::Regular, // a `static` + None => CtfeValidationMode::Const { inner, allow_static_ptrs: false }, + }; + ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)?; + inner = true; + } + }; + let alloc_id = mplace.ptr.provenance.unwrap(); + if let Err(error) = validation { + // Validation failed, report an error. This is always a hard error. + let err = ConstEvalErr::new(&ecx, error, None); + Err(err.struct_error( + ecx.tcx, + "it is undefined behavior to use this value", + |diag| { + if matches!(err.error, InterpError::UndefinedBehavior(_)) { + diag.note(NOTE_ON_UNDEFINED_BEHAVIOR_ERROR); + } + diag.note(&format!( + "the raw bytes of the constant ({}", + display_allocation( + *ecx.tcx, + ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner() + ) + )); + }, + )) + } else { + // Convert to raw constant + Ok(ConstAlloc { alloc_id, ty: mplace.layout.ty }) + } + } + } +} diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs new file mode 100644 index 000000000..f1674d04f --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs @@ -0,0 +1,82 @@ +use rustc_hir as hir; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_middle::ty::query::Providers; +use rustc_middle::ty::{DefIdTree, TyCtxt}; +use rustc_span::symbol::Symbol; + +/// Whether the `def_id` is an unstable const fn and what feature gate is necessary to enable it +pub fn is_unstable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> Option { + if tcx.is_const_fn_raw(def_id) { + let const_stab = tcx.lookup_const_stability(def_id)?; + if const_stab.is_const_unstable() { Some(const_stab.feature) } else { None } + } else { + None + } +} + +pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { + let parent_id = tcx.local_parent(def_id); + tcx.def_kind(parent_id) == DefKind::Impl && tcx.constness(parent_id) == hir::Constness::Const +} + +/// Checks whether an item is considered to be `const`. If it is a constructor, it is const. If +/// it is a trait impl/function, return if it has a `const` modifier. If it is an intrinsic, +/// report whether said intrinsic has a `rustc_const_{un,}stable` attribute. Otherwise, return +/// `Constness::NotConst`. +fn constness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::Constness { + let def_id = def_id.expect_local(); + let node = tcx.hir().get_by_def_id(def_id); + + match node { + hir::Node::Ctor(_) => hir::Constness::Const, + hir::Node::Item(hir::Item { kind: hir::ItemKind::Impl(impl_), .. }) => impl_.constness, + hir::Node::ForeignItem(hir::ForeignItem { kind: hir::ForeignItemKind::Fn(..), .. }) => { + // Intrinsics use `rustc_const_{un,}stable` attributes to indicate constness. All other + // foreign items cannot be evaluated at compile-time. + let is_const = if tcx.is_intrinsic(def_id) { + tcx.lookup_const_stability(def_id).is_some() + } else { + false + }; + if is_const { hir::Constness::Const } else { hir::Constness::NotConst } + } + _ => { + if let Some(fn_kind) = node.fn_kind() { + if fn_kind.constness() == hir::Constness::Const { + return hir::Constness::Const; + } + + // If the function itself is not annotated with `const`, it may still be a `const fn` + // if it resides in a const trait impl. + let is_const = is_parent_const_impl_raw(tcx, def_id); + if is_const { hir::Constness::Const } else { hir::Constness::NotConst } + } else { + hir::Constness::NotConst + } + } + } +} + +fn is_promotable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + tcx.is_const_fn(def_id) + && match tcx.lookup_const_stability(def_id) { + Some(stab) => { + if cfg!(debug_assertions) && stab.promotable { + let sig = tcx.fn_sig(def_id); + assert_eq!( + sig.unsafety(), + hir::Unsafety::Normal, + "don't mark const unsafe fns as promotable", + // https://github.com/rust-lang/rust/pull/53851#issuecomment-418760682 + ); + } + stab.promotable + } + None => false, + } +} + +pub fn provide(providers: &mut Providers) { + *providers = Providers { constness, is_promotable_const_fn, ..*providers }; +} diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs new file mode 100644 index 000000000..fc2e6652a --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -0,0 +1,527 @@ +use rustc_hir::def::DefKind; +use rustc_middle::mir; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use std::borrow::Borrow; +use std::collections::hash_map::Entry; +use std::hash::Hash; + +use rustc_data_structures::fx::FxHashMap; +use std::fmt; + +use rustc_ast::Mutability; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::AssertMessage; +use rustc_session::Limit; +use rustc_span::symbol::{sym, Symbol}; +use rustc_target::abi::{Align, Size}; +use rustc_target::spec::abi::Abi as CallAbi; + +use crate::interpret::{ + self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, + OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, +}; + +use super::error::*; + +impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> { + /// "Intercept" a function call to a panic-related function + /// because we have something special to do for it. + /// If this returns successfully (`Ok`), the function should just be evaluated normally. + fn hook_special_const_fn( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + ) -> InterpResult<'tcx, Option>> { + // All `#[rustc_do_not_const_check]` functions should be hooked here. + let def_id = instance.def_id(); + + if Some(def_id) == self.tcx.lang_items().const_eval_select() { + // redirect to const_eval_select_ct + if let Some(const_eval_select) = self.tcx.lang_items().const_eval_select_ct() { + return Ok(Some( + ty::Instance::resolve( + *self.tcx, + ty::ParamEnv::reveal_all(), + const_eval_select, + instance.substs, + ) + .unwrap() + .unwrap(), + )); + } + } else if Some(def_id) == self.tcx.lang_items().panic_display() + || Some(def_id) == self.tcx.lang_items().begin_panic_fn() + { + // &str or &&str + assert!(args.len() == 1); + + let mut msg_place = self.deref_operand(&args[0])?; + while msg_place.layout.ty.is_ref() { + msg_place = self.deref_operand(&msg_place.into())?; + } + + let msg = Symbol::intern(self.read_str(&msg_place)?); + let span = self.find_closest_untracked_caller_location(); + let (file, line, col) = self.location_triple_for_span(span); + return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into()); + } else if Some(def_id) == self.tcx.lang_items().panic_fmt() { + // For panic_fmt, call const_panic_fmt instead. + if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() { + return Ok(Some( + ty::Instance::resolve( + *self.tcx, + ty::ParamEnv::reveal_all(), + const_panic_fmt, + self.tcx.intern_substs(&[]), + ) + .unwrap() + .unwrap(), + )); + } + } + Ok(None) + } +} + +/// Extra machine state for CTFE, and the Machine instance +pub struct CompileTimeInterpreter<'mir, 'tcx> { + /// For now, the number of terminators that can be evaluated before we throw a resource + /// exhaustion error. + /// + /// Setting this to `0` disables the limit and allows the interpreter to run forever. + pub steps_remaining: usize, + + /// The virtual call stack. + pub(crate) stack: Vec>, + + /// We need to make sure consts never point to anything mutable, even recursively. That is + /// relied on for pattern matching on consts with references. + /// To achieve this, two pieces have to work together: + /// * Interning makes everything outside of statics immutable. + /// * Pointers to allocations inside of statics can never leak outside, to a non-static global. + /// This boolean here controls the second part. + pub(super) can_access_statics: bool, +} + +impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> { + pub(crate) fn new(const_eval_limit: Limit, can_access_statics: bool) -> Self { + CompileTimeInterpreter { + steps_remaining: const_eval_limit.0, + stack: Vec::new(), + can_access_statics, + } + } +} + +impl interpret::AllocMap for FxHashMap { + #[inline(always)] + fn contains_key(&mut self, k: &Q) -> bool + where + K: Borrow, + { + FxHashMap::contains_key(self, k) + } + + #[inline(always)] + fn insert(&mut self, k: K, v: V) -> Option { + FxHashMap::insert(self, k, v) + } + + #[inline(always)] + fn remove(&mut self, k: &Q) -> Option + where + K: Borrow, + { + FxHashMap::remove(self, k) + } + + #[inline(always)] + fn filter_map_collect(&self, mut f: impl FnMut(&K, &V) -> Option) -> Vec { + self.iter().filter_map(move |(k, v)| f(k, &*v)).collect() + } + + #[inline(always)] + fn get_or(&self, k: K, vacant: impl FnOnce() -> Result) -> Result<&V, E> { + match self.get(&k) { + Some(v) => Ok(v), + None => { + vacant()?; + bug!("The CTFE machine shouldn't ever need to extend the alloc_map when reading") + } + } + } + + #[inline(always)] + fn get_mut_or(&mut self, k: K, vacant: impl FnOnce() -> Result) -> Result<&mut V, E> { + match self.entry(k) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let v = vacant()?; + Ok(e.insert(v)) + } + } + } +} + +pub(crate) type CompileTimeEvalContext<'mir, 'tcx> = + InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum MemoryKind { + Heap, +} + +impl fmt::Display for MemoryKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MemoryKind::Heap => write!(f, "heap allocation"), + } + } +} + +impl interpret::MayLeak for MemoryKind { + #[inline(always)] + fn may_leak(self) -> bool { + match self { + MemoryKind::Heap => false, + } + } +} + +impl interpret::MayLeak for ! { + #[inline(always)] + fn may_leak(self) -> bool { + // `self` is uninhabited + self + } +} + +impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { + fn guaranteed_eq(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> { + Ok(match (a, b) { + // Comparisons between integers are always known. + (Scalar::Int { .. }, Scalar::Int { .. }) => a == b, + // Equality with integers can never be known for sure. + (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => false, + // FIXME: return `true` for when both sides are the same pointer, *except* that + // some things (like functions and vtables) do not have stable addresses + // so we need to be careful around them (see e.g. #73722). + (Scalar::Ptr(..), Scalar::Ptr(..)) => false, + }) + } + + fn guaranteed_ne(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> { + Ok(match (a, b) { + // Comparisons between integers are always known. + (Scalar::Int(_), Scalar::Int(_)) => a != b, + // Comparisons of abstract pointers with null pointers are known if the pointer + // is in bounds, because if they are in bounds, the pointer can't be null. + // Inequality with integers other than null can never be known for sure. + (Scalar::Int(int), ptr @ Scalar::Ptr(..)) + | (ptr @ Scalar::Ptr(..), Scalar::Int(int)) => { + int.is_null() && !self.scalar_may_be_null(ptr)? + } + // FIXME: return `true` for at least some comparisons where we can reliably + // determine the result of runtime inequality tests at compile-time. + // Examples include comparison of addresses in different static items. + (Scalar::Ptr(..), Scalar::Ptr(..)) => false, + }) + } +} + +impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, 'tcx> { + compile_time_machine!(<'mir, 'tcx>); + + type MemoryKind = MemoryKind; + + const PANIC_ON_ALLOC_FAIL: bool = false; // will be raised as a proper error + + fn load_mir( + ecx: &InterpCx<'mir, 'tcx, Self>, + instance: ty::InstanceDef<'tcx>, + ) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> { + match instance { + ty::InstanceDef::Item(def) => { + if ecx.tcx.is_ctfe_mir_available(def.did) { + Ok(ecx.tcx.mir_for_ctfe_opt_const_arg(def)) + } else if ecx.tcx.def_kind(def.did) == DefKind::AssocConst { + let guar = ecx.tcx.sess.delay_span_bug( + rustc_span::DUMMY_SP, + "This is likely a const item that is missing from its impl", + ); + throw_inval!(AlreadyReported(guar)); + } else { + let path = ecx.tcx.def_path_str(def.did); + Err(ConstEvalErrKind::NeedsRfc(format!("calling extern function `{}`", path)) + .into()) + } + } + _ => Ok(ecx.tcx.instance_mir(instance)), + } + } + + fn find_mir_or_eval_fn( + ecx: &mut InterpCx<'mir, 'tcx, Self>, + instance: ty::Instance<'tcx>, + _abi: CallAbi, + args: &[OpTy<'tcx>], + _dest: &PlaceTy<'tcx>, + _ret: Option, + _unwind: StackPopUnwind, // unwinding is not supported in consts + ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> { + debug!("find_mir_or_eval_fn: {:?}", instance); + + // Only check non-glue functions + if let ty::InstanceDef::Item(def) = instance.def { + // Execution might have wandered off into other crates, so we cannot do a stability- + // sensitive check here. But we can at least rule out functions that are not const + // at all. + if !ecx.tcx.is_const_fn_raw(def.did) { + // allow calling functions inside a trait marked with #[const_trait]. + if !ecx.tcx.is_const_default_method(def.did) { + // We certainly do *not* want to actually call the fn + // though, so be sure we return here. + throw_unsup_format!("calling non-const function `{}`", instance) + } + } + + if let Some(new_instance) = ecx.hook_special_const_fn(instance, args)? { + // We call another const fn instead. + // However, we return the *original* instance to make backtraces work out + // (and we hope this does not confuse the FnAbi checks too much). + return Ok(Self::find_mir_or_eval_fn( + ecx, + new_instance, + _abi, + args, + _dest, + _ret, + _unwind, + )? + .map(|(body, _instance)| (body, instance))); + } + } + // This is a const fn. Call it. + Ok(Some((ecx.load_mir(instance.def, None)?, instance))) + } + + fn call_intrinsic( + ecx: &mut InterpCx<'mir, 'tcx, Self>, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + dest: &PlaceTy<'tcx, Self::Provenance>, + target: Option, + _unwind: StackPopUnwind, + ) -> InterpResult<'tcx> { + // Shared intrinsics. + if ecx.emulate_intrinsic(instance, args, dest, target)? { + return Ok(()); + } + let intrinsic_name = ecx.tcx.item_name(instance.def_id()); + + // CTFE-specific intrinsics. + let Some(ret) = target else { + return Err(ConstEvalErrKind::NeedsRfc(format!( + "calling intrinsic `{}`", + intrinsic_name + )) + .into()); + }; + match intrinsic_name { + sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => { + let a = ecx.read_immediate(&args[0])?.to_scalar()?; + let b = ecx.read_immediate(&args[1])?.to_scalar()?; + let cmp = if intrinsic_name == sym::ptr_guaranteed_eq { + ecx.guaranteed_eq(a, b)? + } else { + ecx.guaranteed_ne(a, b)? + }; + ecx.write_scalar(Scalar::from_bool(cmp), dest)?; + } + sym::const_allocate => { + let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?; + let align = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?; + + let align = match Align::from_bytes(align) { + Ok(a) => a, + Err(err) => throw_ub_format!("align has to be a power of 2, {}", err), + }; + + let ptr = ecx.allocate_ptr( + Size::from_bytes(size as u64), + align, + interpret::MemoryKind::Machine(MemoryKind::Heap), + )?; + ecx.write_pointer(ptr, dest)?; + } + sym::const_deallocate => { + let ptr = ecx.read_pointer(&args[0])?; + let size = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?; + let align = ecx.read_scalar(&args[2])?.to_machine_usize(ecx)?; + + let size = Size::from_bytes(size); + let align = match Align::from_bytes(align) { + Ok(a) => a, + Err(err) => throw_ub_format!("align has to be a power of 2, {}", err), + }; + + // If an allocation is created in an another const, + // we don't deallocate it. + let (alloc_id, _, _) = ecx.ptr_get_alloc_id(ptr)?; + let is_allocated_in_another_const = matches!( + ecx.tcx.try_get_global_alloc(alloc_id), + Some(interpret::GlobalAlloc::Memory(_)) + ); + + if !is_allocated_in_another_const { + ecx.deallocate_ptr( + ptr, + Some((size, align)), + interpret::MemoryKind::Machine(MemoryKind::Heap), + )?; + } + } + _ => { + return Err(ConstEvalErrKind::NeedsRfc(format!( + "calling intrinsic `{}`", + intrinsic_name + )) + .into()); + } + } + + ecx.go_to_block(ret); + Ok(()) + } + + fn assert_panic( + ecx: &mut InterpCx<'mir, 'tcx, Self>, + msg: &AssertMessage<'tcx>, + _unwind: Option, + ) -> InterpResult<'tcx> { + use rustc_middle::mir::AssertKind::*; + // Convert `AssertKind` to `AssertKind`. + let eval_to_int = + |op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int()); + let err = match msg { + BoundsCheck { ref len, ref index } => { + let len = eval_to_int(len)?; + let index = eval_to_int(index)?; + BoundsCheck { len, index } + } + Overflow(op, l, r) => Overflow(*op, eval_to_int(l)?, eval_to_int(r)?), + OverflowNeg(op) => OverflowNeg(eval_to_int(op)?), + DivisionByZero(op) => DivisionByZero(eval_to_int(op)?), + RemainderByZero(op) => RemainderByZero(eval_to_int(op)?), + ResumedAfterReturn(generator_kind) => ResumedAfterReturn(*generator_kind), + ResumedAfterPanic(generator_kind) => ResumedAfterPanic(*generator_kind), + }; + Err(ConstEvalErrKind::AssertFailure(err).into()) + } + + fn abort(_ecx: &mut InterpCx<'mir, 'tcx, Self>, msg: String) -> InterpResult<'tcx, !> { + Err(ConstEvalErrKind::Abort(msg).into()) + } + + fn binary_ptr_op( + _ecx: &InterpCx<'mir, 'tcx, Self>, + _bin_op: mir::BinOp, + _left: &ImmTy<'tcx>, + _right: &ImmTy<'tcx>, + ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { + Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into()) + } + + fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { + // The step limit has already been hit in a previous call to `before_terminator`. + if ecx.machine.steps_remaining == 0 { + return Ok(()); + } + + ecx.machine.steps_remaining -= 1; + if ecx.machine.steps_remaining == 0 { + throw_exhaust!(StepLimitReached) + } + + Ok(()) + } + + #[inline(always)] + fn expose_ptr( + _ecx: &mut InterpCx<'mir, 'tcx, Self>, + _ptr: Pointer, + ) -> InterpResult<'tcx> { + Err(ConstEvalErrKind::NeedsRfc("exposing pointers".to_string()).into()) + } + + #[inline(always)] + fn init_frame_extra( + ecx: &mut InterpCx<'mir, 'tcx, Self>, + frame: Frame<'mir, 'tcx>, + ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { + // Enforce stack size limit. Add 1 because this is run before the new frame is pushed. + if !ecx.recursion_limit.value_within_limit(ecx.stack().len() + 1) { + throw_exhaust!(StackFrameLimitReached) + } else { + Ok(frame) + } + } + + #[inline(always)] + fn stack<'a>( + ecx: &'a InterpCx<'mir, 'tcx, Self>, + ) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] { + &ecx.machine.stack + } + + #[inline(always)] + fn stack_mut<'a>( + ecx: &'a mut InterpCx<'mir, 'tcx, Self>, + ) -> &'a mut Vec> { + &mut ecx.machine.stack + } + + fn before_access_global( + _tcx: TyCtxt<'tcx>, + machine: &Self, + alloc_id: AllocId, + alloc: ConstAllocation<'tcx>, + static_def_id: Option, + is_write: bool, + ) -> InterpResult<'tcx> { + let alloc = alloc.inner(); + if is_write { + // Write access. These are never allowed, but we give a targeted error message. + if alloc.mutability == Mutability::Not { + Err(err_ub!(WriteToReadOnly(alloc_id)).into()) + } else { + Err(ConstEvalErrKind::ModifiedGlobal.into()) + } + } else { + // Read access. These are usually allowed, with some exceptions. + if machine.can_access_statics { + // Machine configuration allows us read from anything (e.g., `static` initializer). + Ok(()) + } else if static_def_id.is_some() { + // Machine configuration does not allow us to read statics + // (e.g., `const` initializer). + // See const_eval::machine::MemoryExtra::can_access_statics for why + // this check is so important: if we could read statics, we could read pointers + // to mutable allocations *inside* statics. These allocations are not themselves + // statics, so pointers to them can get around the check in `validity.rs`. + Err(ConstEvalErrKind::ConstAccessesStatic.into()) + } else { + // Immutable global, this read is fine. + // But make sure we never accept a read from something mutable, that would be + // unsound. The reason is that as the content of this allocation may be different + // now and at run-time, so if we permit reading now we might return the wrong value. + assert_eq!(alloc.mutability, Mutability::Not); + Ok(()) + } + } + } +} + +// Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups +// so we can end up having a file with just that impl, but for now, let's keep the impl discoverable +// at the bottom of this file. diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs new file mode 100644 index 000000000..948c33494 --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -0,0 +1,163 @@ +// Not in interpret to make sure we do not use private implementation details + +use rustc_hir::Mutability; +use rustc_middle::mir; +use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId}; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::{source_map::DUMMY_SP, symbol::Symbol}; + +use crate::interpret::{ + intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, InterpResult, MemPlaceMeta, + Scalar, +}; + +mod error; +mod eval_queries; +mod fn_queries; +mod machine; +mod valtrees; + +pub use error::*; +pub use eval_queries::*; +pub use fn_queries::*; +pub use machine::*; +pub(crate) use valtrees::{const_to_valtree_inner, valtree_to_const_value}; + +pub(crate) fn const_caller_location( + tcx: TyCtxt<'_>, + (file, line, col): (Symbol, u32, u32), +) -> ConstValue<'_> { + trace!("const_caller_location: {}:{}:{}", file, line, col); + let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), false); + + let loc_place = ecx.alloc_caller_location(file, line, col); + if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() { + bug!("intern_const_alloc_recursive should not error in this case") + } + ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr, &tcx)) +} + +// We forbid type-level constants that contain more than `VALTREE_MAX_NODES` nodes. +const VALTREE_MAX_NODES: usize = 100000; + +pub(crate) enum ValTreeCreationError { + NodesOverflow, + NonSupportedType, + Other, +} +pub(crate) type ValTreeCreationResult<'tcx> = Result, ValTreeCreationError>; + +/// Evaluates a constant and turns it into a type-level constant value. +pub(crate) fn eval_to_valtree<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + cid: GlobalId<'tcx>, +) -> EvalToValTreeResult<'tcx> { + let const_alloc = tcx.eval_to_allocation_raw(param_env.and(cid))?; + + // FIXME Need to provide a span to `eval_to_valtree` + let ecx = mk_eval_cx( + tcx, DUMMY_SP, param_env, + // It is absolutely crucial for soundness that + // we do not read from static items or other mutable memory. + false, + ); + let place = ecx.raw_const_to_mplace(const_alloc).unwrap(); + debug!(?place); + + let mut num_nodes = 0; + let valtree_result = const_to_valtree_inner(&ecx, &place, &mut num_nodes); + + match valtree_result { + Ok(valtree) => Ok(Some(valtree)), + Err(err) => { + let did = cid.instance.def_id(); + let s = cid.display(tcx); + match err { + ValTreeCreationError::NodesOverflow => { + let msg = format!("maximum number of nodes exceeded in constant {}", &s); + let mut diag = match tcx.hir().span_if_local(did) { + Some(span) => tcx.sess.struct_span_err(span, &msg), + None => tcx.sess.struct_err(&msg), + }; + diag.emit(); + + Ok(None) + } + ValTreeCreationError::NonSupportedType | ValTreeCreationError::Other => Ok(None), + } + } + } +} + +#[instrument(skip(tcx), level = "debug")] +pub(crate) fn try_destructure_mir_constant<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + val: mir::ConstantKind<'tcx>, +) -> InterpResult<'tcx, mir::DestructuredMirConstant<'tcx>> { + trace!("destructure_mir_constant: {:?}", val); + let ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false); + let op = ecx.mir_const_to_op(&val, None)?; + + // We go to `usize` as we cannot allocate anything bigger anyway. + let (field_count, variant, down) = match val.ty().kind() { + ty::Array(_, len) => (len.eval_usize(tcx, param_env) as usize, None, op), + ty::Adt(def, _) if def.variants().is_empty() => { + throw_ub!(Unreachable) + } + ty::Adt(def, _) => { + let variant = ecx.read_discriminant(&op)?.1; + let down = ecx.operand_downcast(&op, variant)?; + (def.variants()[variant].fields.len(), Some(variant), down) + } + ty::Tuple(substs) => (substs.len(), None, op), + _ => bug!("cannot destructure mir constant {:?}", val), + }; + + let fields_iter = (0..field_count) + .map(|i| { + let field_op = ecx.operand_field(&down, i)?; + let val = op_to_const(&ecx, &field_op); + Ok(mir::ConstantKind::Val(val, field_op.layout.ty)) + }) + .collect::>>()?; + let fields = tcx.arena.alloc_from_iter(fields_iter); + + Ok(mir::DestructuredMirConstant { variant, fields }) +} + +#[instrument(skip(tcx), level = "debug")] +pub(crate) fn deref_mir_constant<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + val: mir::ConstantKind<'tcx>, +) -> mir::ConstantKind<'tcx> { + let ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false); + let op = ecx.mir_const_to_op(&val, None).unwrap(); + let mplace = ecx.deref_operand(&op).unwrap(); + if let Some(alloc_id) = mplace.ptr.provenance { + assert_eq!( + tcx.global_alloc(alloc_id).unwrap_memory().0.0.mutability, + Mutability::Not, + "deref_mir_constant cannot be used with mutable allocations as \ + that could allow pattern matching to observe mutable statics", + ); + } + + let ty = match mplace.meta { + MemPlaceMeta::None => mplace.layout.ty, + // In case of unsized types, figure out the real type behind. + MemPlaceMeta::Meta(scalar) => match mplace.layout.ty.kind() { + ty::Str => bug!("there's no sized equivalent of a `str`"), + ty::Slice(elem_ty) => tcx.mk_array(*elem_ty, scalar.to_machine_usize(&tcx).unwrap()), + _ => bug!( + "type {} should not have metadata, but had {:?}", + mplace.layout.ty, + mplace.meta + ), + }, + }; + + mir::ConstantKind::Val(op_to_const(&ecx, &mplace.into()), ty) +} diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs new file mode 100644 index 000000000..8fff4571d --- /dev/null +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -0,0 +1,475 @@ +use super::eval_queries::{mk_eval_cx, op_to_const}; +use super::machine::CompileTimeEvalContext; +use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES}; +use crate::interpret::{ + intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta, + MemoryKind, PlaceTy, Scalar, ScalarMaybeUninit, +}; +use crate::interpret::{MPlaceTy, Value}; +use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; +use rustc_span::source_map::DUMMY_SP; +use rustc_target::abi::{Align, VariantIdx}; + +#[instrument(skip(ecx), level = "debug")] +fn branches<'tcx>( + ecx: &CompileTimeEvalContext<'tcx, 'tcx>, + place: &MPlaceTy<'tcx>, + n: usize, + variant: Option, + num_nodes: &mut usize, +) -> ValTreeCreationResult<'tcx> { + let place = match variant { + Some(variant) => ecx.mplace_downcast(&place, variant).unwrap(), + None => *place, + }; + let variant = variant.map(|variant| Some(ty::ValTree::Leaf(ScalarInt::from(variant.as_u32())))); + debug!(?place, ?variant); + + let mut fields = Vec::with_capacity(n); + for i in 0..n { + let field = ecx.mplace_field(&place, i).unwrap(); + let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?; + fields.push(Some(valtree)); + } + + // For enums, we prepend their variant index before the variant's fields so we can figure out + // the variant again when just seeing a valtree. + let branches = variant + .into_iter() + .chain(fields.into_iter()) + .collect::>>() + .expect("should have already checked for errors in ValTree creation"); + + // Have to account for ZSTs here + if branches.len() == 0 { + *num_nodes += 1; + } + + Ok(ty::ValTree::Branch(ecx.tcx.arena.alloc_from_iter(branches))) +} + +#[instrument(skip(ecx), level = "debug")] +fn slice_branches<'tcx>( + ecx: &CompileTimeEvalContext<'tcx, 'tcx>, + place: &MPlaceTy<'tcx>, + num_nodes: &mut usize, +) -> ValTreeCreationResult<'tcx> { + let n = place + .len(&ecx.tcx.tcx) + .unwrap_or_else(|_| panic!("expected to use len of place {:?}", place)); + + let mut elems = Vec::with_capacity(n as usize); + for i in 0..n { + let place_elem = ecx.mplace_index(place, i).unwrap(); + let valtree = const_to_valtree_inner(ecx, &place_elem, num_nodes)?; + elems.push(valtree); + } + + Ok(ty::ValTree::Branch(ecx.tcx.arena.alloc_from_iter(elems))) +} + +#[instrument(skip(ecx), level = "debug")] +pub(crate) fn const_to_valtree_inner<'tcx>( + ecx: &CompileTimeEvalContext<'tcx, 'tcx>, + place: &MPlaceTy<'tcx>, + num_nodes: &mut usize, +) -> ValTreeCreationResult<'tcx> { + let ty = place.layout.ty; + debug!("ty kind: {:?}", ty.kind()); + + if *num_nodes >= VALTREE_MAX_NODES { + return Err(ValTreeCreationError::NodesOverflow); + } + + match ty.kind() { + ty::FnDef(..) => { + *num_nodes += 1; + Ok(ty::ValTree::zst()) + } + ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => { + let Ok(val) = ecx.read_immediate(&place.into()) else { + return Err(ValTreeCreationError::Other); + }; + let val = val.to_scalar().unwrap(); + *num_nodes += 1; + + Ok(ty::ValTree::Leaf(val.assert_int())) + } + + // Raw pointers are not allowed in type level constants, as we cannot properly test them for + // equality at compile-time (see `ptr_guaranteed_eq`/`_ne`). + // Technically we could allow function pointers (represented as `ty::Instance`), but this is not guaranteed to + // agree with runtime equality tests. + ty::FnPtr(_) | ty::RawPtr(_) => Err(ValTreeCreationError::NonSupportedType), + + ty::Ref(_, _, _) => { + let Ok(derefd_place)= ecx.deref_operand(&place.into()) else { + return Err(ValTreeCreationError::Other); + }; + debug!(?derefd_place); + + const_to_valtree_inner(ecx, &derefd_place, num_nodes) + } + + ty::Str | ty::Slice(_) | ty::Array(_, _) => { + slice_branches(ecx, place, num_nodes) + } + // Trait objects are not allowed in type level constants, as we have no concept for + // resolving their backing type, even if we can do that at const eval time. We may + // hypothetically be able to allow `dyn StructuralEq` trait objects in the future, + // but it is unclear if this is useful. + ty::Dynamic(..) => Err(ValTreeCreationError::NonSupportedType), + + ty::Tuple(elem_tys) => { + branches(ecx, place, elem_tys.len(), None, num_nodes) + } + + ty::Adt(def, _) => { + if def.is_union() { + return Err(ValTreeCreationError::NonSupportedType); + } else if def.variants().is_empty() { + bug!("uninhabited types should have errored and never gotten converted to valtree") + } + + let Ok((_, variant)) = ecx.read_discriminant(&place.into()) else { + return Err(ValTreeCreationError::Other); + }; + branches(ecx, place, def.variant(variant).fields.len(), def.is_enum().then_some(variant), num_nodes) + } + + ty::Never + | ty::Error(_) + | ty::Foreign(..) + | ty::Infer(ty::FreshIntTy(_)) + | ty::Infer(ty::FreshFloatTy(_)) + | ty::Projection(..) + | ty::Param(_) + | ty::Bound(..) + | ty::Placeholder(..) + // FIXME(oli-obk): we could look behind opaque types + | ty::Opaque(..) + | ty::Infer(_) + // FIXME(oli-obk): we can probably encode closures just like structs + | ty::Closure(..) + | ty::Generator(..) + | ty::GeneratorWitness(..) => Err(ValTreeCreationError::NonSupportedType), + } +} + +#[instrument(skip(ecx), level = "debug")] +fn create_mplace_from_layout<'tcx>( + ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, + ty: Ty<'tcx>, +) -> MPlaceTy<'tcx> { + let tcx = ecx.tcx; + let param_env = ecx.param_env; + let layout = tcx.layout_of(param_env.and(ty)).unwrap(); + debug!(?layout); + + ecx.allocate(layout, MemoryKind::Stack).unwrap() +} + +// Walks custom DSTs and gets the type of the unsized field and the number of elements +// in the unsized field. +fn get_info_on_unsized_field<'tcx>( + ty: Ty<'tcx>, + valtree: ty::ValTree<'tcx>, + tcx: TyCtxt<'tcx>, +) -> (Ty<'tcx>, usize) { + let mut last_valtree = valtree; + let tail = tcx.struct_tail_with_normalize( + ty, + |ty| ty, + || { + let branches = last_valtree.unwrap_branch(); + last_valtree = branches[branches.len() - 1]; + debug!(?branches, ?last_valtree); + }, + ); + let unsized_inner_ty = match tail.kind() { + ty::Slice(t) => *t, + ty::Str => tail, + _ => bug!("expected Slice or Str"), + }; + + // Have to adjust type for ty::Str + let unsized_inner_ty = match unsized_inner_ty.kind() { + ty::Str => tcx.mk_ty(ty::Uint(ty::UintTy::U8)), + _ => unsized_inner_ty, + }; + + // Get the number of elements in the unsized field + let num_elems = last_valtree.unwrap_branch().len(); + + (unsized_inner_ty, num_elems) +} + +#[instrument(skip(ecx), level = "debug")] +fn create_pointee_place<'tcx>( + ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, + ty: Ty<'tcx>, + valtree: ty::ValTree<'tcx>, +) -> MPlaceTy<'tcx> { + let tcx = ecx.tcx.tcx; + + if !ty.is_sized(ecx.tcx, ty::ParamEnv::empty()) { + // We need to create `Allocation`s for custom DSTs + + let (unsized_inner_ty, num_elems) = get_info_on_unsized_field(ty, valtree, tcx); + let unsized_inner_ty = match unsized_inner_ty.kind() { + ty::Str => tcx.mk_ty(ty::Uint(ty::UintTy::U8)), + _ => unsized_inner_ty, + }; + let unsized_inner_ty_size = + tcx.layout_of(ty::ParamEnv::empty().and(unsized_inner_ty)).unwrap().layout.size(); + debug!(?unsized_inner_ty, ?unsized_inner_ty_size, ?num_elems); + + // for custom DSTs only the last field/element is unsized, but we need to also allocate + // space for the other fields/elements + let layout = tcx.layout_of(ty::ParamEnv::empty().and(ty)).unwrap(); + let size_of_sized_part = layout.layout.size(); + + // Get the size of the memory behind the DST + let dst_size = unsized_inner_ty_size.checked_mul(num_elems as u64, &tcx).unwrap(); + + let size = size_of_sized_part.checked_add(dst_size, &tcx).unwrap(); + let align = Align::from_bytes(size.bytes().next_power_of_two()).unwrap(); + let ptr = ecx.allocate_ptr(size, align, MemoryKind::Stack).unwrap(); + debug!(?ptr); + + let place = MPlaceTy::from_aligned_ptr_with_meta( + ptr.into(), + layout, + MemPlaceMeta::Meta(Scalar::from_machine_usize(num_elems as u64, &tcx)), + ); + debug!(?place); + + place + } else { + create_mplace_from_layout(ecx, ty) + } +} + +/// Converts a `ValTree` to a `ConstValue`, which is needed after mir +/// construction has finished. +// FIXME Merge `valtree_to_const_value` and `valtree_into_mplace` into one function +#[instrument(skip(tcx), level = "debug")] +pub fn valtree_to_const_value<'tcx>( + tcx: TyCtxt<'tcx>, + param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, + valtree: ty::ValTree<'tcx>, +) -> ConstValue<'tcx> { + // Basic idea: We directly construct `Scalar` values from trivial `ValTree`s + // (those for constants with type bool, int, uint, float or char). + // For all other types we create an `MPlace` and fill that by walking + // the `ValTree` and using `place_projection` and `place_field` to + // create inner `MPlace`s which are filled recursively. + // FIXME Does this need an example? + + let (param_env, ty) = param_env_ty.into_parts(); + let mut ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false); + + match ty.kind() { + ty::FnDef(..) => { + assert!(valtree.unwrap_branch().is_empty()); + ConstValue::ZeroSized + } + ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => match valtree { + ty::ValTree::Leaf(scalar_int) => ConstValue::Scalar(Scalar::Int(scalar_int)), + ty::ValTree::Branch(_) => bug!( + "ValTrees for Bool, Int, Uint, Float or Char should have the form ValTree::Leaf" + ), + }, + ty::Ref(_, _, _) | ty::Tuple(_) | ty::Array(_, _) | ty::Adt(..) => { + let mut place = match ty.kind() { + ty::Ref(_, inner_ty, _) => { + // Need to create a place for the pointee to fill for Refs + create_pointee_place(&mut ecx, *inner_ty, valtree) + } + _ => create_mplace_from_layout(&mut ecx, ty), + }; + debug!(?place); + + valtree_into_mplace(&mut ecx, &mut place, valtree); + dump_place(&ecx, place.into()); + intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &place).unwrap(); + + let const_val = match ty.kind() { + ty::Ref(_, _, _) => { + let ref_place = place.to_ref(&tcx); + let imm = + ImmTy::from_immediate(ref_place, tcx.layout_of(param_env_ty).unwrap()); + + op_to_const(&ecx, &imm.into()) + } + _ => op_to_const(&ecx, &place.into()), + }; + debug!(?const_val); + + const_val + } + ty::Never + | ty::Error(_) + | ty::Foreign(..) + | ty::Infer(ty::FreshIntTy(_)) + | ty::Infer(ty::FreshFloatTy(_)) + | ty::Projection(..) + | ty::Param(_) + | ty::Bound(..) + | ty::Placeholder(..) + | ty::Opaque(..) + | ty::Infer(_) + | ty::Closure(..) + | ty::Generator(..) + | ty::GeneratorWitness(..) + | ty::FnPtr(_) + | ty::RawPtr(_) + | ty::Str + | ty::Slice(_) + | ty::Dynamic(..) => bug!("no ValTree should have been created for type {:?}", ty.kind()), + } +} + +#[instrument(skip(ecx), level = "debug")] +fn valtree_into_mplace<'tcx>( + ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, + place: &mut MPlaceTy<'tcx>, + valtree: ty::ValTree<'tcx>, +) { + // This will match on valtree and write the value(s) corresponding to the ValTree + // inside the place recursively. + + let tcx = ecx.tcx.tcx; + let ty = place.layout.ty; + + match ty.kind() { + ty::FnDef(_, _) => { + ecx.write_immediate(Immediate::Uninit, &place.into()).unwrap(); + } + ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => { + let scalar_int = valtree.unwrap_leaf(); + debug!("writing trivial valtree {:?} to place {:?}", scalar_int, place); + ecx.write_immediate( + Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar_int.into())), + &place.into(), + ) + .unwrap(); + } + ty::Ref(_, inner_ty, _) => { + let mut pointee_place = create_pointee_place(ecx, *inner_ty, valtree); + debug!(?pointee_place); + + valtree_into_mplace(ecx, &mut pointee_place, valtree); + dump_place(ecx, pointee_place.into()); + intern_const_alloc_recursive(ecx, InternKind::Constant, &pointee_place).unwrap(); + + let imm = match inner_ty.kind() { + ty::Slice(_) | ty::Str => { + let len = valtree.unwrap_branch().len(); + let len_scalar = + ScalarMaybeUninit::Scalar(Scalar::from_machine_usize(len as u64, &tcx)); + + Immediate::ScalarPair( + ScalarMaybeUninit::from_maybe_pointer((*pointee_place).ptr, &tcx), + len_scalar, + ) + } + _ => pointee_place.to_ref(&tcx), + }; + debug!(?imm); + + ecx.write_immediate(imm, &place.into()).unwrap(); + } + ty::Adt(_, _) | ty::Tuple(_) | ty::Array(_, _) | ty::Str | ty::Slice(_) => { + let branches = valtree.unwrap_branch(); + + // Need to downcast place for enums + let (place_adjusted, branches, variant_idx) = match ty.kind() { + ty::Adt(def, _) if def.is_enum() => { + // First element of valtree corresponds to variant + let scalar_int = branches[0].unwrap_leaf(); + let variant_idx = VariantIdx::from_u32(scalar_int.try_to_u32().unwrap()); + let variant = def.variant(variant_idx); + debug!(?variant); + + ( + place.project_downcast(ecx, variant_idx).unwrap(), + &branches[1..], + Some(variant_idx), + ) + } + _ => (*place, branches, None), + }; + debug!(?place_adjusted, ?branches); + + // Create the places (by indexing into `place`) for the fields and fill + // them recursively + for (i, inner_valtree) in branches.iter().enumerate() { + debug!(?i, ?inner_valtree); + + let mut place_inner = match ty.kind() { + ty::Str | ty::Slice(_) => ecx.mplace_index(&place, i as u64).unwrap(), + _ if !ty.is_sized(ecx.tcx, ty::ParamEnv::empty()) + && i == branches.len() - 1 => + { + // Note: For custom DSTs we need to manually process the last unsized field. + // We created a `Pointer` for the `Allocation` of the complete sized version of + // the Adt in `create_pointee_place` and now we fill that `Allocation` with the + // values in the ValTree. For the unsized field we have to additionally add the meta + // data. + + let (unsized_inner_ty, num_elems) = + get_info_on_unsized_field(ty, valtree, tcx); + debug!(?unsized_inner_ty); + + let inner_ty = match ty.kind() { + ty::Adt(def, substs) => { + def.variant(VariantIdx::from_u32(0)).fields[i].ty(tcx, substs) + } + ty::Tuple(inner_tys) => inner_tys[i], + _ => bug!("unexpected unsized type {:?}", ty), + }; + + let inner_layout = + tcx.layout_of(ty::ParamEnv::empty().and(inner_ty)).unwrap(); + debug!(?inner_layout); + + let offset = place_adjusted.layout.fields.offset(i); + place + .offset_with_meta( + offset, + MemPlaceMeta::Meta(Scalar::from_machine_usize( + num_elems as u64, + &tcx, + )), + inner_layout, + &tcx, + ) + .unwrap() + } + _ => ecx.mplace_field(&place_adjusted, i).unwrap(), + }; + + debug!(?place_inner); + valtree_into_mplace(ecx, &mut place_inner, *inner_valtree); + dump_place(&ecx, place_inner.into()); + } + + debug!("dump of place_adjusted:"); + dump_place(ecx, place_adjusted.into()); + + if let Some(variant_idx) = variant_idx { + // don't forget filling the place with the discriminant of the enum + ecx.write_discriminant(variant_idx, &place.into()).unwrap(); + } + + debug!("dump of place after writing discriminant:"); + dump_place(ecx, place.into()); + } + _ => bug!("shouldn't have created a ValTree for {:?}", ty), + } +} + +fn dump_place<'tcx>(ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: PlaceTy<'tcx>) { + trace!("{:?}", ecx.dump_place(*place)); +} -- cgit v1.2.3