diff options
Diffstat (limited to 'compiler/rustc_const_eval')
29 files changed, 861 insertions, 776 deletions
diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs index 322bfd5ce..09d53331b 100644 --- a/compiler/rustc_const_eval/src/const_eval/error.rs +++ b/compiler/rustc_const_eval/src/const_eval/error.rs @@ -9,13 +9,13 @@ use rustc_span::{Span, Symbol}; use super::InterpCx; use crate::interpret::{ - struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType, UnsupportedOpInfo, + struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType, + UnsupportedOpInfo, }; /// The CTFE machine has some custom error kinds. #[derive(Clone, Debug)] pub enum ConstEvalErrKind { - NeedsRfc(String), ConstAccessesStatic, ModifiedGlobal, AssertFailure(AssertKind<ConstInt>), @@ -42,9 +42,6 @@ impl fmt::Display for ConstEvalErrKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use self::ConstEvalErrKind::*; match *self { - NeedsRfc(ref msg) => { - write!(f, "\"{}\" needs an rfc before being allowed inside constants", msg) - } ConstAccessesStatic => write!(f, "constant accesses static"), ModifiedGlobal => { write!(f, "modifying a static's initial value from another static's initializer") @@ -158,6 +155,7 @@ impl<'tcx> ConstEvalErr<'tcx> { InterpError::Unsupported( UnsupportedOpInfo::ReadPointerAsBytes | UnsupportedOpInfo::PartialPointerOverwrite(_) + | UnsupportedOpInfo::PartialPointerCopy(_), ) => { err.help("this code performed an operation that depends on the underlying bytes representing a pointer"); err.help("the absolute address of a pointer is not known at compile-time, so such operations are not supported"); diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 975fb4b22..a2f14e753 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -2,8 +2,8 @@ use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr}; use crate::interpret::eval_nullary_intrinsic; use crate::interpret::{ intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, - Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, - ScalarMaybeUninit, StackPopCleanup, InterpError, + Immediate, InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy, + RefTracking, StackPopCleanup, }; use rustc_hir::def::DefKind; @@ -74,14 +74,16 @@ fn eval_body_using_ecx<'mir, 'tcx>( None => InternKind::Constant, } }; + ecx.machine.check_alignment = false; // interning doesn't need to respect alignment intern_const_alloc_recursive(ecx, intern_kind, &ret)?; + // we leave alignment checks off, since this `ecx` will not be used for further evaluation anyway debug!("eval_body_using_ecx done: {:?}", *ret); Ok(ret) } /// The `InterpCx` is only meant to be used to do field and index projections into constants for -/// `simd_shuffle` and const patterns in match arms. +/// `simd_shuffle` and const patterns in match arms. It never performs alignment checks. /// /// The function containing the `match` that is currently being analyzed may have generic bounds /// that inform us about the generic bounds of the constant. E.g., using an associated constant @@ -98,7 +100,11 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>( tcx, root_span, param_env, - CompileTimeInterpreter::new(tcx.const_eval_limit(), can_access_statics), + CompileTimeInterpreter::new( + tcx.const_eval_limit(), + can_access_statics, + /*check_alignment:*/ false, + ), ) } @@ -166,10 +172,7 @@ pub(super) fn op_to_const<'tcx>( // see comment on `let try_as_immediate` above Err(imm) => match *imm { _ if imm.layout.is_zst() => ConstValue::ZeroSized, - Immediate::Scalar(x) => match x { - ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s), - ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place()), - }, + Immediate::Scalar(x) => ConstValue::Scalar(x), Immediate::ScalarPair(a, b) => { debug!("ScalarPair(a: {:?}, b: {:?})", a, b); // We know `offset` is relative to the allocation, so we can use `into_parts`. @@ -194,7 +197,7 @@ pub(super) fn op_to_const<'tcx>( } } -#[instrument(skip(tcx), level = "debug")] +#[instrument(skip(tcx), level = "debug", ret)] pub(crate) fn turn_into_const_value<'tcx>( tcx: TyCtxt<'tcx>, constant: ConstAlloc<'tcx>, @@ -203,7 +206,13 @@ pub(crate) fn turn_into_const_value<'tcx>( let cid = key.value; let def_id = cid.instance.def.def_id(); let is_static = tcx.is_static(def_id); - let ecx = mk_eval_cx(tcx, tcx.def_span(key.value.instance.def_id()), key.param_env, is_static); + // This is just accessing an already computed constant, so no need to check alginment here. + let ecx = mk_eval_cx( + tcx, + tcx.def_span(key.value.instance.def_id()), + key.param_env, + /*can_access_statics:*/ is_static, + ); let mplace = ecx.raw_const_to_mplace(constant).expect( "can only fail if layout computation failed, \ @@ -215,10 +224,7 @@ pub(crate) fn turn_into_const_value<'tcx>( ); // Turn this into a proper constant. - let const_val = op_to_const(&ecx, &mplace.into()); - debug!(?const_val); - - const_val + op_to_const(&ecx, &mplace.into()) } #[instrument(skip(tcx), level = "debug")] @@ -300,7 +306,11 @@ pub fn eval_to_allocation_raw_provider<'tcx>( key.param_env, // Statics (and promoteds inside statics) may access other statics, because unlike consts // they do not have to behave "as if" they were evaluated at runtime. - CompileTimeInterpreter::new(tcx.const_eval_limit(), /*can_access_statics:*/ is_static), + CompileTimeInterpreter::new( + tcx.const_eval_limit(), + /*can_access_statics:*/ is_static, + /*check_alignment:*/ tcx.sess.opts.unstable_opts.extra_const_ub_checks, + ), ); let res = ecx.load_mir(cid.instance.def, cid.promoted); diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index fc2e6652a..e5acacd91 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -35,21 +35,7 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> { // All `#[rustc_do_not_const_check]` functions should be hooked here. let def_id = instance.def_id(); - if Some(def_id) == self.tcx.lang_items().const_eval_select() { - // redirect to const_eval_select_ct - if let Some(const_eval_select) = self.tcx.lang_items().const_eval_select_ct() { - return Ok(Some( - ty::Instance::resolve( - *self.tcx, - ty::ParamEnv::reveal_all(), - const_eval_select, - instance.substs, - ) - .unwrap() - .unwrap(), - )); - } - } else if Some(def_id) == self.tcx.lang_items().panic_display() + if Some(def_id) == self.tcx.lang_items().panic_display() || Some(def_id) == self.tcx.lang_items().begin_panic_fn() { // &str or &&str @@ -89,10 +75,10 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> { /// exhaustion error. /// /// Setting this to `0` disables the limit and allows the interpreter to run forever. - pub steps_remaining: usize, + pub(super) steps_remaining: usize, /// The virtual call stack. - pub(crate) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>, + pub(super) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>, /// We need to make sure consts never point to anything mutable, even recursively. That is /// relied on for pattern matching on consts with references. @@ -101,14 +87,22 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> { /// * Pointers to allocations inside of statics can never leak outside, to a non-static global. /// This boolean here controls the second part. pub(super) can_access_statics: bool, + + /// Whether to check alignment during evaluation. + pub(super) check_alignment: bool, } impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> { - pub(crate) fn new(const_eval_limit: Limit, can_access_statics: bool) -> Self { + pub(crate) fn new( + const_eval_limit: Limit, + can_access_statics: bool, + check_alignment: bool, + ) -> Self { CompileTimeInterpreter { steps_remaining: const_eval_limit.0, stack: Vec::new(), can_access_statics, + check_alignment, } } } @@ -197,34 +191,35 @@ impl interpret::MayLeak for ! { } impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { - fn guaranteed_eq(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> { + /// See documentation on the `ptr_guaranteed_cmp` intrinsic. + fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> { Ok(match (a, b) { // Comparisons between integers are always known. - (Scalar::Int { .. }, Scalar::Int { .. }) => a == b, - // Equality with integers can never be known for sure. - (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => false, - // FIXME: return `true` for when both sides are the same pointer, *except* that - // some things (like functions and vtables) do not have stable addresses - // so we need to be careful around them (see e.g. #73722). - (Scalar::Ptr(..), Scalar::Ptr(..)) => false, - }) - } - - fn guaranteed_ne(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> { - Ok(match (a, b) { - // Comparisons between integers are always known. - (Scalar::Int(_), Scalar::Int(_)) => a != b, + (Scalar::Int { .. }, Scalar::Int { .. }) => { + if a == b { + 1 + } else { + 0 + } + } // Comparisons of abstract pointers with null pointers are known if the pointer // is in bounds, because if they are in bounds, the pointer can't be null. // Inequality with integers other than null can never be known for sure. (Scalar::Int(int), ptr @ Scalar::Ptr(..)) - | (ptr @ Scalar::Ptr(..), Scalar::Int(int)) => { - int.is_null() && !self.scalar_may_be_null(ptr)? + | (ptr @ Scalar::Ptr(..), Scalar::Int(int)) + if int.is_null() && !self.scalar_may_be_null(ptr)? => + { + 0 } - // FIXME: return `true` for at least some comparisons where we can reliably + // Equality with integers can never be known for sure. + (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => 2, + // FIXME: return a `1` for when both sides are the same pointer, *except* that + // some things (like functions and vtables) do not have stable addresses + // so we need to be careful around them (see e.g. #73722). + // FIXME: return `0` for at least some comparisons where we can reliably // determine the result of runtime inequality tests at compile-time. // Examples include comparison of addresses in different static items. - (Scalar::Ptr(..), Scalar::Ptr(..)) => false, + (Scalar::Ptr(..), Scalar::Ptr(..)) => 2, }) } } @@ -236,6 +231,16 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, const PANIC_ON_ALLOC_FAIL: bool = false; // will be raised as a proper error + #[inline(always)] + fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + ecx.machine.check_alignment + } + + #[inline(always)] + fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks + } + fn load_mir( ecx: &InterpCx<'mir, 'tcx, Self>, instance: ty::InstanceDef<'tcx>, @@ -251,9 +256,10 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, ); throw_inval!(AlreadyReported(guar)); } else { + // `find_mir_or_eval_fn` checks that this is a const fn before even calling us, + // so this should be unreachable. let path = ecx.tcx.def_path_str(def.did); - Err(ConstEvalErrKind::NeedsRfc(format!("calling extern function `{}`", path)) - .into()) + bug!("trying to call extern function `{path}` at compile-time"); } } _ => Ok(ecx.tcx.instance_mir(instance)), @@ -321,22 +327,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, // CTFE-specific intrinsics. let Some(ret) = target else { - return Err(ConstEvalErrKind::NeedsRfc(format!( - "calling intrinsic `{}`", - intrinsic_name - )) - .into()); + throw_unsup_format!("intrinsic `{intrinsic_name}` is not supported at compile-time"); }; match intrinsic_name { - sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => { - let a = ecx.read_immediate(&args[0])?.to_scalar()?; - let b = ecx.read_immediate(&args[1])?.to_scalar()?; - let cmp = if intrinsic_name == sym::ptr_guaranteed_eq { - ecx.guaranteed_eq(a, b)? - } else { - ecx.guaranteed_ne(a, b)? - }; - ecx.write_scalar(Scalar::from_bool(cmp), dest)?; + sym::ptr_guaranteed_cmp => { + let a = ecx.read_scalar(&args[0])?; + let b = ecx.read_scalar(&args[1])?; + let cmp = ecx.guaranteed_cmp(a, b)?; + ecx.write_scalar(Scalar::from_u8(cmp), dest)?; } sym::const_allocate => { let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?; @@ -382,11 +380,9 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, } } _ => { - return Err(ConstEvalErrKind::NeedsRfc(format!( - "calling intrinsic `{}`", - intrinsic_name - )) - .into()); + throw_unsup_format!( + "intrinsic `{intrinsic_name}` is not supported at compile-time" + ); } } @@ -429,7 +425,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, _left: &ImmTy<'tcx>, _right: &ImmTy<'tcx>, ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { - Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into()) + throw_unsup_format!("pointer arithmetic or comparison is not supported at compile-time"); } fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { @@ -451,7 +447,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, _ecx: &mut InterpCx<'mir, 'tcx, Self>, _ptr: Pointer<AllocId>, ) -> InterpResult<'tcx> { - Err(ConstEvalErrKind::NeedsRfc("exposing pointers".to_string()).into()) + // This is only reachable with -Zunleash-the-miri-inside-of-you. + throw_unsup_format!("exposing pointers is not possible at compile-time") } #[inline(always)] diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs index 948c33494..d9c4ae4d5 100644 --- a/compiler/rustc_const_eval/src/const_eval/mod.rs +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -1,16 +1,16 @@ // Not in interpret to make sure we do not use private implementation details +use crate::errors::MaxNumNodesInConstErr; +use crate::interpret::{ + intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, InterpResult, MemPlaceMeta, + Scalar, +}; use rustc_hir::Mutability; use rustc_middle::mir; use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId}; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::{source_map::DUMMY_SP, symbol::Symbol}; -use crate::interpret::{ - intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, InterpResult, MemPlaceMeta, - Scalar, -}; - mod error; mod eval_queries; mod fn_queries; @@ -72,12 +72,17 @@ pub(crate) fn eval_to_valtree<'tcx>( Ok(valtree) => Ok(Some(valtree)), Err(err) => { let did = cid.instance.def_id(); - let s = cid.display(tcx); + let global_const_id = cid.display(tcx); match err { ValTreeCreationError::NodesOverflow => { - let msg = format!("maximum number of nodes exceeded in constant {}", &s); + let msg = format!( + "maximum number of nodes exceeded in constant {}", + &global_const_id + ); let mut diag = match tcx.hir().span_if_local(did) { - Some(span) => tcx.sess.struct_span_err(span, &msg), + Some(span) => { + tcx.sess.create_err(MaxNumNodesInConstErr { span, global_const_id }) + } None => tcx.sess.struct_err(&msg), }; diag.emit(); diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index 8fff4571d..a964fe846 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -3,7 +3,7 @@ use super::machine::CompileTimeEvalContext; use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES}; use crate::interpret::{ intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta, - MemoryKind, PlaceTy, Scalar, ScalarMaybeUninit, + MemoryKind, PlaceTy, Scalar, }; use crate::interpret::{MPlaceTy, Value}; use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; @@ -90,14 +90,14 @@ pub(crate) fn const_to_valtree_inner<'tcx>( let Ok(val) = ecx.read_immediate(&place.into()) else { return Err(ValTreeCreationError::Other); }; - let val = val.to_scalar().unwrap(); + let val = val.to_scalar(); *num_nodes += 1; Ok(ty::ValTree::Leaf(val.assert_int())) } // Raw pointers are not allowed in type level constants, as we cannot properly test them for - // equality at compile-time (see `ptr_guaranteed_eq`/`_ne`). + // equality at compile-time (see `ptr_guaranteed_cmp`). // Technically we could allow function pointers (represented as `ty::Instance`), but this is not guaranteed to // agree with runtime equality tests. ty::FnPtr(_) | ty::RawPtr(_) => Err(ValTreeCreationError::NonSupportedType), @@ -204,7 +204,7 @@ fn get_info_on_unsized_field<'tcx>( (unsized_inner_ty, num_elems) } -#[instrument(skip(ecx), level = "debug")] +#[instrument(skip(ecx), level = "debug", ret)] fn create_pointee_place<'tcx>( ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, ty: Ty<'tcx>, @@ -237,14 +237,11 @@ fn create_pointee_place<'tcx>( let ptr = ecx.allocate_ptr(size, align, MemoryKind::Stack).unwrap(); debug!(?ptr); - let place = MPlaceTy::from_aligned_ptr_with_meta( + MPlaceTy::from_aligned_ptr_with_meta( ptr.into(), layout, MemPlaceMeta::Meta(Scalar::from_machine_usize(num_elems as u64, &tcx)), - ); - debug!(?place); - - place + ) } else { create_mplace_from_layout(ecx, ty) } @@ -253,7 +250,7 @@ fn create_pointee_place<'tcx>( /// Converts a `ValTree` to a `ConstValue`, which is needed after mir /// construction has finished. // FIXME Merge `valtree_to_const_value` and `valtree_into_mplace` into one function -#[instrument(skip(tcx), level = "debug")] +#[instrument(skip(tcx), level = "debug", ret)] pub fn valtree_to_const_value<'tcx>( tcx: TyCtxt<'tcx>, param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, @@ -294,7 +291,7 @@ pub fn valtree_to_const_value<'tcx>( dump_place(&ecx, place.into()); intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &place).unwrap(); - let const_val = match ty.kind() { + match ty.kind() { ty::Ref(_, _, _) => { let ref_place = place.to_ref(&tcx); let imm = @@ -303,10 +300,7 @@ pub fn valtree_to_const_value<'tcx>( op_to_const(&ecx, &imm.into()) } _ => op_to_const(&ecx, &place.into()), - }; - debug!(?const_val); - - const_val + } } ty::Never | ty::Error(_) @@ -349,11 +343,7 @@ fn valtree_into_mplace<'tcx>( ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => { let scalar_int = valtree.unwrap_leaf(); debug!("writing trivial valtree {:?} to place {:?}", scalar_int, place); - ecx.write_immediate( - Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar_int.into())), - &place.into(), - ) - .unwrap(); + ecx.write_immediate(Immediate::Scalar(scalar_int.into()), &place.into()).unwrap(); } ty::Ref(_, inner_ty, _) => { let mut pointee_place = create_pointee_place(ecx, *inner_ty, valtree); @@ -366,11 +356,10 @@ fn valtree_into_mplace<'tcx>( let imm = match inner_ty.kind() { ty::Slice(_) | ty::Str => { let len = valtree.unwrap_branch().len(); - let len_scalar = - ScalarMaybeUninit::Scalar(Scalar::from_machine_usize(len as u64, &tcx)); + let len_scalar = Scalar::from_machine_usize(len as u64, &tcx); Immediate::ScalarPair( - ScalarMaybeUninit::from_maybe_pointer((*pointee_place).ptr, &tcx), + Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx), len_scalar, ) } diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs index a463fe7b9..c3547cb3a 100644 --- a/compiler/rustc_const_eval/src/errors.rs +++ b/compiler/rustc_const_eval/src/errors.rs @@ -3,7 +3,7 @@ use rustc_macros::SessionDiagnostic; use rustc_span::Span; #[derive(SessionDiagnostic)] -#[error(const_eval::unstable_in_stable)] +#[diag(const_eval::unstable_in_stable)] pub(crate) struct UnstableInStable { pub gate: String, #[primary_span] @@ -22,14 +22,14 @@ pub(crate) struct UnstableInStable { } #[derive(SessionDiagnostic)] -#[error(const_eval::thread_local_access, code = "E0625")] +#[diag(const_eval::thread_local_access, code = "E0625")] pub(crate) struct NonConstOpErr { #[primary_span] pub span: Span, } #[derive(SessionDiagnostic)] -#[error(const_eval::static_access, code = "E0013")] +#[diag(const_eval::static_access, code = "E0013")] #[help] pub(crate) struct StaticAccessErr { #[primary_span] @@ -41,7 +41,7 @@ pub(crate) struct StaticAccessErr { } #[derive(SessionDiagnostic)] -#[error(const_eval::raw_ptr_to_int)] +#[diag(const_eval::raw_ptr_to_int)] #[note] #[note(const_eval::note2)] pub(crate) struct RawPtrToIntErr { @@ -50,7 +50,7 @@ pub(crate) struct RawPtrToIntErr { } #[derive(SessionDiagnostic)] -#[error(const_eval::raw_ptr_comparison)] +#[diag(const_eval::raw_ptr_comparison)] #[note] pub(crate) struct RawPtrComparisonErr { #[primary_span] @@ -58,14 +58,14 @@ pub(crate) struct RawPtrComparisonErr { } #[derive(SessionDiagnostic)] -#[error(const_eval::panic_non_str)] +#[diag(const_eval::panic_non_str)] pub(crate) struct PanicNonStrErr { #[primary_span] pub span: Span, } #[derive(SessionDiagnostic)] -#[error(const_eval::mut_deref, code = "E0658")] +#[diag(const_eval::mut_deref, code = "E0658")] pub(crate) struct MutDerefErr { #[primary_span] pub span: Span, @@ -73,7 +73,7 @@ pub(crate) struct MutDerefErr { } #[derive(SessionDiagnostic)] -#[error(const_eval::transient_mut_borrow, code = "E0658")] +#[diag(const_eval::transient_mut_borrow, code = "E0658")] pub(crate) struct TransientMutBorrowErr { #[primary_span] pub span: Span, @@ -81,9 +81,116 @@ pub(crate) struct TransientMutBorrowErr { } #[derive(SessionDiagnostic)] -#[error(const_eval::transient_mut_borrow_raw, code = "E0658")] +#[diag(const_eval::transient_mut_borrow_raw, code = "E0658")] pub(crate) struct TransientMutBorrowErrRaw { #[primary_span] pub span: Span, pub kind: ConstContext, } + +#[derive(SessionDiagnostic)] +#[diag(const_eval::max_num_nodes_in_const)] +pub(crate) struct MaxNumNodesInConstErr { + #[primary_span] + pub span: Span, + pub global_const_id: String, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_fn_pointer_call)] +pub(crate) struct UnallowedFnPointerCall { + #[primary_span] + pub span: Span, + pub kind: ConstContext, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unstable_const_fn)] +pub(crate) struct UnstableConstFn { + #[primary_span] + pub span: Span, + pub def_path: String, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_mutable_refs, code = "E0764")] +pub(crate) struct UnallowedMutableRefs { + #[primary_span] + pub span: Span, + pub kind: ConstContext, + #[note(const_eval::teach_note)] + pub teach: Option<()>, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_mutable_refs_raw, code = "E0764")] +pub(crate) struct UnallowedMutableRefsRaw { + #[primary_span] + pub span: Span, + pub kind: ConstContext, + #[note(const_eval::teach_note)] + pub teach: Option<()>, +} +#[derive(SessionDiagnostic)] +#[diag(const_eval::non_const_fmt_macro_call, code = "E0015")] +pub(crate) struct NonConstFmtMacroCall { + #[primary_span] + pub span: Span, + pub kind: ConstContext, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::non_const_fn_call, code = "E0015")] +pub(crate) struct NonConstFnCall { + #[primary_span] + pub span: Span, + pub def_path_str: String, + pub kind: ConstContext, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_op_in_const_context)] +pub(crate) struct UnallowedOpInConstContext { + #[primary_span] + pub span: Span, + pub msg: String, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_heap_allocations, code = "E0010")] +pub(crate) struct UnallowedHeapAllocations { + #[primary_span] + #[label] + pub span: Span, + pub kind: ConstContext, + #[note(const_eval::teach_note)] + pub teach: Option<()>, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::unallowed_inline_asm, code = "E0015")] +pub(crate) struct UnallowedInlineAsm { + #[primary_span] + pub span: Span, + pub kind: ConstContext, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::interior_mutable_data_refer, code = "E0492")] +pub(crate) struct InteriorMutableDataRefer { + #[primary_span] + #[label] + pub span: Span, + #[help] + pub opt_help: Option<()>, + pub kind: ConstContext, + #[note(const_eval::teach_note)] + pub teach: Option<()>, +} + +#[derive(SessionDiagnostic)] +#[diag(const_eval::interior_mutability_borrow)] +pub(crate) struct InteriorMutabilityBorrow { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index c97c31eb9..cbe985480 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -108,6 +108,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty), } } + + DynStar => { + if let ty::Dynamic(data, _, ty::DynStar) = cast_ty.kind() { + // Initial cast from sized to dyn trait + let vtable = self.get_vtable_ptr(src.layout.ty, data.principal())?; + let vtable = Scalar::from_maybe_pointer(vtable, self); + let data = self.read_immediate(src)?.to_scalar(); + let _assert_pointer_sized = data.to_pointer(self)?; + let val = Immediate::ScalarPair(data, vtable); + self.write_immediate(val, dest)?; + } else { + bug!() + } + } } Ok(()) } @@ -123,10 +137,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { match src.layout.ty.kind() { // Floating point Float(FloatTy::F32) => { - return Ok(self.cast_from_float(src.to_scalar()?.to_f32()?, cast_ty).into()); + return Ok(self.cast_from_float(src.to_scalar().to_f32()?, cast_ty).into()); } Float(FloatTy::F64) => { - return Ok(self.cast_from_float(src.to_scalar()?.to_f64()?, cast_ty).into()); + return Ok(self.cast_from_float(src.to_scalar().to_f64()?, cast_ty).into()); } // The rest is integer/pointer-"like", including fn ptr casts _ => assert!( @@ -153,7 +167,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert_eq!(dest_layout.size, self.pointer_size()); assert!(src.layout.ty.is_unsafe_ptr()); return match **src { - Immediate::ScalarPair(data, _) => Ok(data.check_init()?.into()), + Immediate::ScalarPair(data, _) => Ok(data.into()), Immediate::Scalar(..) => span_bug!( self.cur_span(), "{:?} input to a fat-to-thin cast ({:?} -> {:?})", @@ -167,7 +181,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } // # The remaining source values are scalar and "int-like". - let scalar = src.to_scalar()?; + let scalar = src.to_scalar(); Ok(self.cast_from_int_like(scalar, src.layout, cast_ty)?.into()) } @@ -179,7 +193,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert_matches!(src.layout.ty.kind(), ty::RawPtr(_) | ty::FnPtr(_)); assert!(cast_ty.is_integral()); - let scalar = src.to_scalar()?; + let scalar = src.to_scalar(); let ptr = scalar.to_pointer(self)?; match ptr.into_pointer_or_addr() { Ok(ptr) => M::expose_ptr(self, ptr)?, @@ -197,7 +211,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert_matches!(cast_ty.kind(), ty::RawPtr(_)); // First cast to usize. - let scalar = src.to_scalar()?; + let scalar = src.to_scalar(); let addr = self.cast_from_int_like(scalar, src.layout, self.tcx.types.usize)?; let addr = addr.to_machine_usize(self)?; @@ -291,14 +305,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { match (&src_pointee_ty.kind(), &dest_pointee_ty.kind()) { (&ty::Array(_, length), &ty::Slice(_)) => { - let ptr = self.read_immediate(src)?.to_scalar()?; + let ptr = self.read_scalar(src)?; // u64 cast is from usize to u64, which is always good let val = Immediate::new_slice(ptr, length.eval_usize(*self.tcx, self.param_env), self); self.write_immediate(val, dest) } (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => { - let (old_data, old_vptr) = self.read_immediate(src)?.to_scalar_pair()?; + let val = self.read_immediate(src)?; + if data_a.principal() == data_b.principal() { + // A NOP cast that doesn't actually change anything, should be allowed even with mismatching vtables. + return self.write_immediate(*val, dest); + } + let (old_data, old_vptr) = val.to_scalar_pair(); let old_vptr = old_vptr.to_pointer(self)?; let (ty, old_trait) = self.get_ptr_vtable(old_vptr)?; if old_trait != data_a.principal() { @@ -307,10 +326,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let new_vptr = self.get_vtable_ptr(ty, data_b.principal())?; self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest) } - (_, &ty::Dynamic(ref data, _)) => { + (_, &ty::Dynamic(ref data, _, ty::Dyn)) => { // Initial cast from sized to dyn trait let vtable = self.get_vtable_ptr(src_pointee_ty, data.principal())?; - let ptr = self.read_immediate(src)?.to_scalar()?; + let ptr = self.read_scalar(src)?; let val = Immediate::new_dyn_trait(ptr, vtable, &*self.tcx); self.write_immediate(val, dest) } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 150d6589b..d37eaeed0 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -21,7 +21,7 @@ use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayou use super::{ AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance, - Scalar, ScalarMaybeUninit, StackPopJump, + Scalar, StackPopJump, }; use crate::transform::validate::equal_up_to_regions; @@ -187,9 +187,6 @@ pub enum LocalValue<Prov: Provenance = AllocId> { impl<'tcx, Prov: Provenance + 'static> LocalState<'tcx, Prov> { /// Read the local's value or error if the local is not yet live or not live anymore. - /// - /// Note: This may only be invoked from the `Machine::access_local` hook and not from - /// anywhere else. You may be invalidating machine invariants if you do! #[inline] pub fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> { match &self.value { @@ -782,7 +779,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert_eq!( unwinding, match self.frame().loc { - Ok(loc) => self.body().basic_blocks()[loc.block].is_cleanup, + Ok(loc) => self.body().basic_blocks[loc.block].is_cleanup, Err(_) => true, } ); @@ -991,16 +988,16 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> std::fmt::Debug } LocalValue::Live(Operand::Immediate(Immediate::Scalar(val))) => { write!(fmt, " {:?}", val)?; - if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val { + if let Scalar::Ptr(ptr, _size) = val { allocs.push(ptr.provenance.get_alloc_id()); } } LocalValue::Live(Operand::Immediate(Immediate::ScalarPair(val1, val2))) => { write!(fmt, " ({:?}, {:?})", val1, val2)?; - if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val1 { + if let Scalar::Ptr(ptr, _size) = val1 { allocs.push(ptr.provenance.get_alloc_id()); } - if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val2 { + if let Scalar::Ptr(ptr, _size) = val2 { allocs.push(ptr.provenance.get_alloc_id()); } } diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index 376b8872c..24dbc7695 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -134,7 +134,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval: alloc.mutability = Mutability::Not; }; // link the alloc id to the actual allocation - leftover_allocations.extend(alloc.relocations().iter().map(|&(_, alloc_id)| alloc_id)); + leftover_allocations.extend(alloc.provenance().iter().map(|&(_, alloc_id)| alloc_id)); let alloc = tcx.intern_const_alloc(alloc); tcx.set_alloc_id_memory(alloc_id, alloc); None @@ -191,10 +191,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory return Ok(true); }; - // If there are no relocations in this allocation, it does not contain references + // If there is no provenance in this allocation, it does not contain references // that point to another allocation, and we can avoid the interning walk. if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? { - if !alloc.has_relocations() { + if !alloc.has_provenance() { return Ok(false); } } else { @@ -233,8 +233,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory } fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> { - // Handle Reference types, as these are the only relocations supported by const eval. - // Raw pointers (and boxes) are handled by the `leftover_relocations` logic. + // Handle Reference types, as these are the only types with provenance supported by const eval. + // Raw pointers (and boxes) are handled by the `leftover_allocations` logic. let tcx = self.ecx.tcx; let ty = mplace.layout.ty; if let ty::Ref(_, referenced_ty, ref_mutability) = *ty.kind() { @@ -334,7 +334,7 @@ pub enum InternKind { /// tracks where in the value we are and thus can show much better error messages. /// Any errors here would anyway be turned into `const_err` lints, whereas validation failures /// are hard errors. -#[tracing::instrument(level = "debug", skip(ecx))] +#[instrument(level = "debug", skip(ecx))] pub fn intern_const_alloc_recursive< 'mir, 'tcx: 'mir, @@ -410,7 +410,7 @@ pub fn intern_const_alloc_recursive< // references and a `leftover_allocations` set (where we only have a todo-list here). // So we hand-roll the interning logic here again. match intern_kind { - // Statics may contain mutable allocations even behind relocations. + // Statics may point to mutable allocations. // Even for immutable statics it would be ok to have mutable allocations behind // raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`. InternKind::Static(_) => {} @@ -441,7 +441,7 @@ pub fn intern_const_alloc_recursive< } let alloc = tcx.intern_const_alloc(alloc); tcx.set_alloc_id_memory(alloc_id, alloc); - for &(_, alloc_id) in alloc.inner().relocations().iter() { + for &(_, alloc_id) in alloc.inner().provenance().iter() { if leftover_allocations.insert(alloc_id) { todo.push(alloc_id); } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 08209eb79..8637d6a77 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -8,7 +8,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::mir::{ self, interpret::{ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar}, - BinOp, + BinOp, NonDivergingIntrinsic, }; use rustc_middle::ty; use rustc_middle::ty::layout::LayoutOf as _; @@ -79,9 +79,9 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>( ty::Projection(_) | ty::Opaque(_, _) | ty::Param(_) - | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => throw_inval!(TooGeneric), + ty::Bound(_, _) => bug!("bound ty during ctfe"), ty::Bool | ty::Char | ty::Int(_) @@ -95,7 +95,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>( | ty::Ref(_, _, _) | ty::FnDef(_, _) | ty::FnPtr(_) - | ty::Dynamic(_, _) + | ty::Dynamic(_, _, _) | ty::Closure(_, _) | ty::Generator(_, _, _) | ty::GeneratorWitness(_) @@ -184,7 +184,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | sym::bitreverse => { let ty = substs.type_at(0); let layout_of = self.layout_of(ty)?; - let val = self.read_scalar(&args[0])?.check_init()?; + let val = self.read_scalar(&args[0])?; let bits = val.to_bits(layout_of.size)?; let kind = match layout_of.abi { Abi::Scalar(scalar) => scalar.primitive(), @@ -256,7 +256,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let (val, overflowed, _ty) = self.overflowing_binary_op(bin_op, &l, &r)?; if overflowed { let layout = self.layout_of(substs.type_at(0))?; - let r_val = r.to_scalar()?.to_bits(layout.size)?; + let r_val = r.to_scalar().to_bits(layout.size)?; if let sym::unchecked_shl | sym::unchecked_shr = intrinsic_name { throw_ub_format!("overflowing shift by {} in `{}`", r_val, intrinsic_name); } else { @@ -269,9 +269,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW)) // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW)) let layout = self.layout_of(substs.type_at(0))?; - let val = self.read_scalar(&args[0])?.check_init()?; + let val = self.read_scalar(&args[0])?; let val_bits = val.to_bits(layout.size)?; - let raw_shift = self.read_scalar(&args[1])?.check_init()?; + let raw_shift = self.read_scalar(&args[1])?; let raw_shift_bits = raw_shift.to_bits(layout.size)?; let width_bits = u128::from(layout.size.bits()); let shift_bits = raw_shift_bits % width_bits; @@ -320,7 +320,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let (a_offset, b_offset) = match (self.ptr_try_get_alloc_id(a), self.ptr_try_get_alloc_id(b)) { (Err(a), Err(b)) => { - // Neither poiner points to an allocation. + // Neither pointer points to an allocation. // If these are inequal or null, this *will* fail the deref check below. (a, b) } @@ -506,12 +506,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // These just return their argument self.copy_op(&args[0], dest, /*allow_transmute*/ false)?; } - sym::assume => { - let cond = self.read_scalar(&args[0])?.check_init()?.to_bool()?; - if !cond { - throw_ub_format!("`assume` intrinsic called with `false`"); - } - } sym::raw_eq => { let result = self.raw_eq_intrinsic(&args[0], &args[1])?; self.write_scalar(result, dest)?; @@ -536,6 +530,32 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Ok(true) } + pub(super) fn emulate_nondiverging_intrinsic( + &mut self, + intrinsic: &NonDivergingIntrinsic<'tcx>, + ) -> InterpResult<'tcx> { + match intrinsic { + NonDivergingIntrinsic::Assume(op) => { + let op = self.eval_operand(op, None)?; + let cond = self.read_scalar(&op)?.to_bool()?; + if !cond { + throw_ub_format!("`assume` called with `false`"); + } + Ok(()) + } + NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { + count, + src, + dst, + }) => { + let src = self.eval_operand(src, None)?; + let dst = self.eval_operand(dst, None)?; + let count = self.eval_operand(count, None)?; + self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true) + } + } + } + pub fn exact_div( &mut self, a: &ImmTy<'tcx, M::Provenance>, @@ -570,7 +590,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // term since the sign of the second term can be inferred from this and // the fact that the operation has overflowed (if either is 0 no // overflow can occur) - let first_term: u128 = l.to_scalar()?.to_bits(l.layout.size)?; + let first_term: u128 = l.to_scalar().to_bits(l.layout.size)?; let first_term_positive = first_term & (1 << (num_bits - 1)) == 0; if first_term_positive { // Negative overflow not possible since the positive first term @@ -687,10 +707,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap().ty)?; assert!(!layout.is_unsized()); - let lhs = self.read_pointer(lhs)?; - let rhs = self.read_pointer(rhs)?; - let lhs_bytes = self.read_bytes_ptr(lhs, layout.size)?; - let rhs_bytes = self.read_bytes_ptr(rhs, layout.size)?; + let get_bytes = |this: &InterpCx<'mir, 'tcx, M>, + op: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, + size| + -> InterpResult<'tcx, &[u8]> { + let ptr = this.read_pointer(op)?; + let Some(alloc_ref) = self.get_ptr_alloc(ptr, size, Align::ONE)? else { + // zero-sized access + return Ok(&[]); + }; + if alloc_ref.has_provenance() { + throw_ub_format!("`raw_eq` on bytes with provenance"); + } + alloc_ref.get_bytes_strip_provenance() + }; + + let lhs_bytes = get_bytes(self, lhs, layout.size)?; + let rhs_bytes = get_bytes(self, rhs, layout.size)?; Ok(Scalar::from_bool(lhs_bytes == rhs_bytes)) } } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs index 5864b9215..91f4f0425 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs @@ -28,7 +28,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let mut source_info = *frame.body.source_info(loc); // If this is a `Call` terminator, use the `fn_span` instead. - let block = &frame.body.basic_blocks()[loc.block]; + let block = &frame.body.basic_blocks[loc.block]; if loc.statement_index == block.statements.len() { debug!( "find_closest_untracked_caller_location: got terminator {:?} ({:?})", diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs index f9847742f..7e4c5fcb0 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs @@ -48,7 +48,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { | ty::FnPtr(_) | ty::Never | ty::Tuple(_) - | ty::Dynamic(_, _) => self.pretty_print_type(ty), + | ty::Dynamic(_, _, _) => self.pretty_print_type(ty), // Placeholders (all printed as `_` to uniformize them). ty::Param(_) | ty::Bound(..) | ty::Placeholder(_) | ty::Infer(_) | ty::Error(_) => { diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 71ccd1799..530e252b7 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -6,6 +6,7 @@ use std::borrow::{Borrow, Cow}; use std::fmt::Debug; use std::hash::Hash; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_middle::mir; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::DefId; @@ -123,18 +124,15 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Whether memory accesses should be alignment-checked. fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; - /// Whether, when checking alignment, we should `force_int` and thus support + /// Whether, when checking alignment, we should look at the actual address and thus support /// custom alignment logic based on whatever the integer address happens to be. /// - /// Requires Provenance::OFFSET_IS_ADDR to be true. - fn force_int_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; + /// If this returns true, Provenance::OFFSET_IS_ADDR must be true. + fn use_addr_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; /// Whether to enforce the validity invariant fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; - /// Whether to enforce integers and floats being initialized. - fn enforce_number_init(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; - /// Whether function calls should be [ABI](CallAbi)-checked. fn enforce_abi(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { true @@ -218,23 +216,12 @@ pub trait Machine<'mir, 'tcx>: Sized { right: &ImmTy<'tcx, Self::Provenance>, ) -> InterpResult<'tcx, (Scalar<Self::Provenance>, bool, Ty<'tcx>)>; - /// Called to read the specified `local` from the `frame`. - /// Since reading a ZST is not actually accessing memory or locals, this is never invoked - /// for ZST reads. - #[inline] - fn access_local<'a>( - frame: &'a Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>, - local: mir::Local, - ) -> InterpResult<'tcx, &'a Operand<Self::Provenance>> - where - 'tcx: 'mir, - { - frame.locals[local].access() - } - /// Called to write the specified `local` from the `frame`. /// Since writing a ZST is not actually accessing memory or locals, this is never invoked /// for ZST reads. + /// + /// Due to borrow checker trouble, we indicate the `frame` as an index rather than an `&mut + /// Frame`. #[inline] fn access_local_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, @@ -329,7 +316,7 @@ pub trait Machine<'mir, 'tcx>: Sized { /// cache the result. (This relies on `AllocMap::get_or` being able to add the /// owned allocation to the map even when the map is shared.) /// - /// This must only fail if `alloc` contains relocations. + /// This must only fail if `alloc` contains provenance. fn adjust_allocation<'b>( ecx: &InterpCx<'mir, 'tcx, Self>, id: AllocId, @@ -337,13 +324,22 @@ pub trait Machine<'mir, 'tcx>: Sized { kind: Option<MemoryKind<Self::MemoryKind>>, ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra>>>; + fn eval_inline_asm( + _ecx: &mut InterpCx<'mir, 'tcx, Self>, + _template: &'tcx [InlineAsmTemplatePiece], + _operands: &[mir::InlineAsmOperand<'tcx>], + _options: InlineAsmOptions, + ) -> InterpResult<'tcx> { + throw_unsup_format!("inline assembly is not supported") + } + /// Hook for performing extra checks on a memory read access. /// /// Takes read-only access to the allocation so we can keep all the memory read /// operations take `&self`. Use a `RefCell` in `AllocExtra` if you /// need to mutate. #[inline(always)] - fn memory_read( + fn before_memory_read( _tcx: TyCtxt<'tcx>, _machine: &Self, _alloc_extra: &Self::AllocExtra, @@ -355,7 +351,7 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Hook for performing extra checks on a memory write access. #[inline(always)] - fn memory_written( + fn before_memory_write( _tcx: TyCtxt<'tcx>, _machine: &mut Self, _alloc_extra: &mut Self::AllocExtra, @@ -367,7 +363,7 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Hook for performing extra operations on a memory deallocation. #[inline(always)] - fn memory_deallocated( + fn before_memory_deallocation( _tcx: TyCtxt<'tcx>, _machine: &mut Self, _alloc_extra: &mut Self::AllocExtra, @@ -437,29 +433,12 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { type FrameExtra = (); #[inline(always)] - fn enforce_alignment(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { - // We do not check for alignment to avoid having to carry an `Align` - // in `ConstValue::ByRef`. - false - } - - #[inline(always)] - fn force_int_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { - // We do not support `force_int`. + fn use_addr_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { + // We do not support `use_addr`. false } #[inline(always)] - fn enforce_validity(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { - false // for now, we don't enforce validity - } - - #[inline(always)] - fn enforce_number_init(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { - true - } - - #[inline(always)] fn checked_binop_checks_overflow(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { true } @@ -510,6 +489,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { ) -> InterpResult<$tcx, Pointer<Option<AllocId>>> { // Allow these casts, but make the pointer not dereferenceable. // (I.e., they behave like transmutation.) + // This is correct because no pointers can ever be exposed in compile-time evaluation. Ok(Pointer::from_addr(addr)) } diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index ed2c4edf9..ed155fbfe 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -21,7 +21,6 @@ use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar, - ScalarMaybeUninit, }; #[derive(Debug, PartialEq, Copy, Clone)] @@ -215,7 +214,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.allocate_raw_ptr(alloc, kind).unwrap() } - /// This can fail only of `alloc` contains relocations. + /// This can fail only of `alloc` contains provenance. pub fn allocate_raw_ptr( &mut self, alloc: Allocation, @@ -327,7 +326,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Let the machine take some extra action let size = alloc.size(); - M::memory_deallocated( + M::before_memory_deallocation( *self.tcx, &mut self.machine, &mut alloc.extra, @@ -438,15 +437,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { msg, }) } - // Ensure we never consider the null pointer dereferencable. + // Ensure we never consider the null pointer dereferenceable. if M::Provenance::OFFSET_IS_ADDR { assert_ne!(ptr.addr(), Size::ZERO); } // Test align. Check this last; if both bounds and alignment are violated // we want the error to be about the bounds. if let Some(align) = align { - if M::force_int_for_alignment_check(self) { - // `force_int_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true. + if M::use_addr_for_alignment_check(self) { + // `use_addr_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true. check_offset_align(ptr.addr().bytes(), align)?; } else { // Check allocation alignment and offset alignment. @@ -520,6 +519,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Gives raw access to the `Allocation`, without bounds or alignment checks. /// The caller is responsible for calling the access hooks! + /// + /// You almost certainly want to use `get_ptr_alloc`/`get_ptr_alloc_mut` instead. fn get_alloc_raw( &self, id: AllocId, @@ -573,7 +574,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { )?; if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc { let range = alloc_range(offset, size); - M::memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?; + M::before_memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?; Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id })) } else { // Even in this branch we have to be sure that we actually access the allocation, in @@ -589,6 +590,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Ok(&self.get_alloc_raw(id)?.extra) } + /// Return the `mutability` field of the given allocation. + pub fn get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability> { + Ok(self.get_alloc_raw(id)?.mutability) + } + /// Gives raw mutable access to the `Allocation`, without bounds or alignment checks. /// The caller is responsible for calling the access hooks! /// @@ -634,7 +640,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`. let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?; let range = alloc_range(offset, size); - M::memory_written(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?; + M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?; Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id })) } else { Ok(None) @@ -788,10 +794,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { todo.extend(static_roots); while let Some(id) = todo.pop() { if reachable.insert(id) { - // This is a new allocation, add its relocations to `todo`. + // This is a new allocation, add the allocation it points to to `todo`. if let Some((_, alloc)) = self.memory.alloc_map.get(id) { todo.extend( - alloc.relocations().values().filter_map(|prov| prov.get_alloc_id()), + alloc.provenance().values().filter_map(|prov| prov.get_alloc_id()), ); } } @@ -827,7 +833,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, allocs_to_print: &mut VecDeque<AllocId>, alloc: &Allocation<Prov, Extra>, ) -> std::fmt::Result { - for alloc_id in alloc.relocations().values().filter_map(|prov| prov.get_alloc_id()) { + for alloc_id in alloc.provenance().values().filter_map(|prov| prov.get_alloc_id()) { allocs_to_print.push_back(alloc_id); } write!(fmt, "{}", display_allocation(tcx, alloc)) @@ -894,11 +900,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, /// Reading and writing. impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> { /// `range` is relative to this allocation reference, not the base of the allocation. - pub fn write_scalar( - &mut self, - range: AllocRange, - val: ScalarMaybeUninit<Prov>, - ) -> InterpResult<'tcx> { + pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> { let range = self.range.subrange(range); debug!("write_scalar at {:?}{range:?}: {val:?}", self.alloc_id); Ok(self @@ -908,15 +910,11 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> { } /// `offset` is relative to this allocation reference, not the base of the allocation. - pub fn write_ptr_sized( - &mut self, - offset: Size, - val: ScalarMaybeUninit<Prov>, - ) -> InterpResult<'tcx> { + pub fn write_ptr_sized(&mut self, offset: Size, val: Scalar<Prov>) -> InterpResult<'tcx> { self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size), val) } - /// Mark the entire referenced range as uninitalized + /// Mark the entire referenced range as uninitialized pub fn write_uninit(&mut self) -> InterpResult<'tcx> { Ok(self .alloc @@ -931,7 +929,7 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> { &self, range: AllocRange, read_provenance: bool, - ) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> { + ) -> InterpResult<'tcx, Scalar<Prov>> { let range = self.range.subrange(range); let res = self .alloc @@ -942,12 +940,12 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> { } /// `range` is relative to this allocation reference, not the base of the allocation. - pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> { + pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, Scalar<Prov>> { self.read_scalar(range, /*read_provenance*/ false) } /// `offset` is relative to this allocation reference, not the base of the allocation. - pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> { + pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, Scalar<Prov>> { self.read_scalar( alloc_range(offset, self.tcx.data_layout().pointer_size), /*read_provenance*/ true, @@ -955,29 +953,25 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> { } /// `range` is relative to this allocation reference, not the base of the allocation. - pub fn check_bytes( - &self, - range: AllocRange, - allow_uninit: bool, - allow_ptr: bool, - ) -> InterpResult<'tcx> { + pub fn get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]> { Ok(self .alloc - .check_bytes(&self.tcx, self.range.subrange(range), allow_uninit, allow_ptr) + .get_bytes_strip_provenance(&self.tcx, self.range) .map_err(|e| e.to_interp_error(self.alloc_id))?) } - /// Returns whether the allocation has relocations for the entire range of the `AllocRef`. - pub(crate) fn has_relocations(&self) -> bool { - self.alloc.has_relocations(&self.tcx, self.range) + /// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`. + pub(crate) fn has_provenance(&self) -> bool { + self.alloc.range_has_provenance(&self.tcx, self.range) } } impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { - /// Reads the given number of bytes from memory. Returns them as a slice. + /// Reads the given number of bytes from memory, and strips their provenance if possible. + /// Returns them as a slice. /// /// Performs appropriate bounds checks. - pub fn read_bytes_ptr( + pub fn read_bytes_ptr_strip_provenance( &self, ptr: Pointer<Option<M::Provenance>>, size: Size, @@ -990,7 +984,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // (We are staying inside the bounds here so all is good.) Ok(alloc_ref .alloc - .get_bytes(&alloc_ref.tcx, alloc_ref.range) + .get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range) .map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?) } @@ -1071,7 +1065,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; let src_alloc = self.get_alloc_raw(src_alloc_id)?; let src_range = alloc_range(src_offset, size); - M::memory_read(*tcx, &self.machine, &src_alloc.extra, (src_alloc_id, src_prov), src_range)?; + M::before_memory_read( + *tcx, + &self.machine, + &src_alloc.extra, + (src_alloc_id, src_prov), + src_range, + )?; // We need the `dest` ptr for the next operation, so we get it now. // We already did the source checks and called the hooks so we are good to return early. let Some((dest_alloc_id, dest_offset, dest_prov)) = dest_parts else { @@ -1079,24 +1079,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { return Ok(()); }; - // This checks relocation edges on the src, which needs to happen before - // `prepare_relocation_copy`. - let src_bytes = src_alloc - .get_bytes_with_uninit_and_ptr(&tcx, src_range) - .map_err(|e| e.to_interp_error(src_alloc_id))? - .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation - // first copy the relocations to a temporary buffer, because - // `get_bytes_mut` will clear the relocations, which is correct, - // since we don't want to keep any relocations at the target. - let relocations = - src_alloc.prepare_relocation_copy(self, src_range, dest_offset, num_copies); + // Checks provenance edges on the src, which needs to happen before + // `prepare_provenance_copy`. + if src_alloc.range_has_provenance(&tcx, alloc_range(src_range.start, Size::ZERO)) { + throw_unsup!(PartialPointerCopy(Pointer::new(src_alloc_id, src_range.start))); + } + if src_alloc.range_has_provenance(&tcx, alloc_range(src_range.end(), Size::ZERO)) { + throw_unsup!(PartialPointerCopy(Pointer::new(src_alloc_id, src_range.end()))); + } + let src_bytes = src_alloc.get_bytes_unchecked(src_range).as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation + // first copy the provenance to a temporary buffer, because + // `get_bytes_mut` will clear the provenance, which is correct, + // since we don't want to keep any provenance at the target. + let provenance = + src_alloc.prepare_provenance_copy(self, src_range, dest_offset, num_copies); // Prepare a copy of the initialization mask. let compressed = src_alloc.compress_uninit_range(src_range); // Destination alloc preparations and access hooks. let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?; let dest_range = alloc_range(dest_offset, size * num_copies); - M::memory_written( + M::before_memory_write( *tcx, extra, &mut dest_alloc.extra, @@ -1118,7 +1121,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { dest_alloc .write_uninit(&tcx, dest_range) .map_err(|e| e.to_interp_error(dest_alloc_id))?; - // We can forget about the relocations, this is all not initialized anyway. + // We can forget about the provenance, this is all not initialized anyway. return Ok(()); } @@ -1162,8 +1165,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`) num_copies, ); - // copy the relocations to the destination - dest_alloc.mark_relocation_range(relocations); + // copy the provenance to the destination + dest_alloc.mark_provenance_range(provenance); Ok(()) } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 94ba62c16..0e3959b61 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -1,11 +1,9 @@ //! Functions concerning immediate values and operands, and reading from operands. //! All high-level functions to read from memory work on operands as sources. -use std::fmt::Write; - use rustc_hir::def::Namespace; use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout}; -use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter, Printer}; +use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter}; use rustc_middle::ty::{ConstInt, DelaySpanBugEmitted, Ty}; use rustc_middle::{mir, ty}; use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding}; @@ -14,7 +12,7 @@ use rustc_target::abi::{VariantIdx, Variants}; use super::{ alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Place, PlaceTy, Pointer, - Provenance, Scalar, ScalarMaybeUninit, + Provenance, Scalar, }; /// An `Immediate` represents a single immediate self-contained Rust value. @@ -27,23 +25,14 @@ use super::{ #[derive(Copy, Clone, Debug)] pub enum Immediate<Prov: Provenance = AllocId> { /// A single scalar value (must have *initialized* `Scalar` ABI). - /// FIXME: we also currently often use this for ZST. - /// `ScalarMaybeUninit` should reject ZST, and we should use `Uninit` for them instead. - Scalar(ScalarMaybeUninit<Prov>), + Scalar(Scalar<Prov>), /// A pair of two scalar value (must have `ScalarPair` ABI where both fields are /// `Scalar::Initialized`). - ScalarPair(ScalarMaybeUninit<Prov>, ScalarMaybeUninit<Prov>), + ScalarPair(Scalar<Prov>, Scalar<Prov>), /// A value of fully uninitialized memory. Can have and size and layout. Uninit, } -impl<Prov: Provenance> From<ScalarMaybeUninit<Prov>> for Immediate<Prov> { - #[inline(always)] - fn from(val: ScalarMaybeUninit<Prov>) -> Self { - Immediate::Scalar(val) - } -} - impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> { #[inline(always)] fn from(val: Scalar<Prov>) -> Self { @@ -51,13 +40,13 @@ impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> { } } -impl<'tcx, Prov: Provenance> Immediate<Prov> { +impl<Prov: Provenance> Immediate<Prov> { pub fn from_pointer(p: Pointer<Prov>, cx: &impl HasDataLayout) -> Self { - Immediate::Scalar(ScalarMaybeUninit::from_pointer(p, cx)) + Immediate::Scalar(Scalar::from_pointer(p, cx)) } pub fn from_maybe_pointer(p: Pointer<Option<Prov>>, cx: &impl HasDataLayout) -> Self { - Immediate::Scalar(ScalarMaybeUninit::from_maybe_pointer(p, cx)) + Immediate::Scalar(Scalar::from_maybe_pointer(p, cx)) } pub fn new_slice(val: Scalar<Prov>, len: u64, cx: &impl HasDataLayout) -> Self { @@ -69,41 +58,28 @@ impl<'tcx, Prov: Provenance> Immediate<Prov> { vtable: Pointer<Option<Prov>>, cx: &impl HasDataLayout, ) -> Self { - Immediate::ScalarPair(val.into(), ScalarMaybeUninit::from_maybe_pointer(vtable, cx)) + Immediate::ScalarPair(val.into(), Scalar::from_maybe_pointer(vtable, cx)) } #[inline] #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn to_scalar_or_uninit(self) -> ScalarMaybeUninit<Prov> { + pub fn to_scalar(self) -> Scalar<Prov> { match self { Immediate::Scalar(val) => val, Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"), - Immediate::Uninit => ScalarMaybeUninit::Uninit, + Immediate::Uninit => bug!("Got uninit where a scalar was expected"), } } #[inline] #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn to_scalar(self) -> InterpResult<'tcx, Scalar<Prov>> { - self.to_scalar_or_uninit().check_init() - } - - #[inline] - #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn to_scalar_or_uninit_pair(self) -> (ScalarMaybeUninit<Prov>, ScalarMaybeUninit<Prov>) { + pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) { match self { Immediate::ScalarPair(val1, val2) => (val1, val2), Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"), - Immediate::Uninit => (ScalarMaybeUninit::Uninit, ScalarMaybeUninit::Uninit), + Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"), } } - - #[inline] - #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn to_scalar_pair(self) -> InterpResult<'tcx, (Scalar<Prov>, Scalar<Prov>)> { - let (val1, val2) = self.to_scalar_or_uninit_pair(); - Ok((val1.check_init()?, val2.check_init()?)) - } } // ScalarPair needs a type to interpret, so we often have an immediate and a type together @@ -119,27 +95,17 @@ impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> { /// Helper function for printing a scalar to a FmtPrinter fn p<'a, 'tcx, Prov: Provenance>( cx: FmtPrinter<'a, 'tcx>, - s: ScalarMaybeUninit<Prov>, + s: Scalar<Prov>, ty: Ty<'tcx>, ) -> Result<FmtPrinter<'a, 'tcx>, std::fmt::Error> { match s { - ScalarMaybeUninit::Scalar(Scalar::Int(int)) => { - cx.pretty_print_const_scalar_int(int, ty, true) - } - ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _sz)) => { + Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true), + Scalar::Ptr(ptr, _sz) => { // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to // print what is points to, which would fail since it has no access to the local // memory. cx.pretty_print_const_pointer(ptr, ty, true) } - ScalarMaybeUninit::Uninit => cx.typed_value( - |mut this| { - this.write_str("uninit ")?; - Ok(this) - }, - |this| this.print_type(ty), - " ", - ), } } ty::tls::with(|tcx| { @@ -269,7 +235,7 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { #[inline] pub fn to_const_int(self) -> ConstInt { assert!(self.layout.ty.is_integral()); - let int = self.to_scalar().expect("to_const_int doesn't work on scalar pairs").assert_int(); + let int = self.to_scalar().assert_int(); ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral()) } } @@ -327,7 +293,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { fn read_immediate_from_mplace_raw( &self, mplace: &MPlaceTy<'tcx, M::Provenance>, - force: bool, ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> { if mplace.layout.is_unsized() { // Don't touch unsized @@ -345,47 +310,44 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // case where some of the bytes are initialized and others are not. So, we need an extra // check that walks over the type of `mplace` to make sure it is truly correct to treat this // like a `Scalar` (or `ScalarPair`). - let scalar_layout = match mplace.layout.abi { - // `if` does not work nested inside patterns, making this a bit awkward to express. - Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => Some(s), - Abi::Scalar(s) if force => Some(s.primitive()), - _ => None, - }; - if let Some(s) = scalar_layout { - let size = s.size(self); - assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size"); - let scalar = alloc - .read_scalar(alloc_range(Size::ZERO, size), /*read_provenance*/ s.is_ptr())?; - return Ok(Some(ImmTy { imm: scalar.into(), layout: mplace.layout })); - } - let scalar_pair_layout = match mplace.layout.abi { + Ok(match mplace.layout.abi { + Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => { + let size = s.size(self); + assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size"); + let scalar = alloc.read_scalar( + alloc_range(Size::ZERO, size), + /*read_provenance*/ s.is_ptr(), + )?; + Some(ImmTy { imm: scalar.into(), layout: mplace.layout }) + } Abi::ScalarPair( abi::Scalar::Initialized { value: a, .. }, abi::Scalar::Initialized { value: b, .. }, - ) => Some((a, b)), - Abi::ScalarPair(a, b) if force => Some((a.primitive(), b.primitive())), - _ => None, - }; - if let Some((a, b)) = scalar_pair_layout { - // We checked `ptr_align` above, so all fields will have the alignment they need. - // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`, - // which `ptr.offset(b_offset)` cannot possibly fail to satisfy. - let (a_size, b_size) = (a.size(self), b.size(self)); - let b_offset = a_size.align_to(b.align(self).abi); - assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields - let a_val = alloc.read_scalar( - alloc_range(Size::ZERO, a_size), - /*read_provenance*/ a.is_ptr(), - )?; - let b_val = alloc - .read_scalar(alloc_range(b_offset, b_size), /*read_provenance*/ b.is_ptr())?; - return Ok(Some(ImmTy { - imm: Immediate::ScalarPair(a_val, b_val), - layout: mplace.layout, - })); - } - // Neither a scalar nor scalar pair. - return Ok(None); + ) => { + // We checked `ptr_align` above, so all fields will have the alignment they need. + // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`, + // which `ptr.offset(b_offset)` cannot possibly fail to satisfy. + let (a_size, b_size) = (a.size(self), b.size(self)); + let b_offset = a_size.align_to(b.align(self).abi); + assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields + let a_val = alloc.read_scalar( + alloc_range(Size::ZERO, a_size), + /*read_provenance*/ a.is_ptr(), + )?; + let b_val = alloc.read_scalar( + alloc_range(b_offset, b_size), + /*read_provenance*/ b.is_ptr(), + )?; + Some(ImmTy { + imm: Immediate::ScalarPair(a_val.into(), b_val.into()), + layout: mplace.layout, + }) + } + _ => { + // Neither a scalar nor scalar pair. + None + } + }) } /// Try returning an immediate for the operand. If the layout does not permit loading this as an @@ -394,20 +356,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// succeed! Whether it succeeds depends on whether the layout can be represented /// in an `Immediate`, not on which data is stored there currently. /// - /// If `force` is `true`, then even scalars with fields that can be ununit will be - /// read. This means the load is lossy and should not be written back! - /// This flag exists only for validity checking. - /// /// This is an internal function that should not usually be used; call `read_immediate` instead. /// ConstProp needs it, though. pub fn read_immediate_raw( &self, src: &OpTy<'tcx, M::Provenance>, - force: bool, ) -> InterpResult<'tcx, Result<ImmTy<'tcx, M::Provenance>, MPlaceTy<'tcx, M::Provenance>>> { Ok(match src.try_as_mplace() { Ok(ref mplace) => { - if let Some(val) = self.read_immediate_from_mplace_raw(mplace, force)? { + if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? { Ok(val) } else { Err(*mplace) @@ -418,24 +375,33 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } /// Read an immediate from a place, asserting that that is possible with the given layout. + /// + /// If this suceeds, the `ImmTy` is never `Uninit`. #[inline(always)] pub fn read_immediate( &self, op: &OpTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { - if let Ok(imm) = self.read_immediate_raw(op, /*force*/ false)? { - Ok(imm) - } else { - span_bug!(self.cur_span(), "primitive read failed for type: {:?}", op.layout.ty); + if !matches!( + op.layout.abi, + Abi::Scalar(abi::Scalar::Initialized { .. }) + | Abi::ScalarPair(abi::Scalar::Initialized { .. }, abi::Scalar::Initialized { .. }) + ) { + span_bug!(self.cur_span(), "primitive read not possible for type: {:?}", op.layout.ty); } + let imm = self.read_immediate_raw(op)?.unwrap(); + if matches!(*imm, Immediate::Uninit) { + throw_ub!(InvalidUninitBytes(None)); + } + Ok(imm) } /// Read a scalar from a place pub fn read_scalar( &self, op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, ScalarMaybeUninit<M::Provenance>> { - Ok(self.read_immediate(op)?.to_scalar_or_uninit()) + ) -> InterpResult<'tcx, Scalar<M::Provenance>> { + Ok(self.read_immediate(op)?.to_scalar()) } /// Read a pointer from a place. @@ -449,7 +415,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Turn the wide MPlace into a string (must already be dereferenced!) pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> { let len = mplace.len(self)?; - let bytes = self.read_bytes_ptr(mplace.ptr, Size::from_bytes(len))?; + let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len))?; let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?; Ok(str) } @@ -478,7 +444,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - /// Read from a local. Will not actually access the local if reading from a ZST. + /// Read from a local. /// Will not access memory, instead an indirect `Operand` is returned. /// /// This is public because it is used by [priroda](https://github.com/oli-obk/priroda) to get an @@ -490,12 +456,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { layout: Option<TyAndLayout<'tcx>>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { let layout = self.layout_of_local(frame, local, layout)?; - let op = if layout.is_zst() { - // Bypass `access_local` (helps in ConstProp) - Operand::Immediate(Immediate::Uninit) - } else { - *M::access_local(frame, local)? - }; + let op = *frame.locals[local].access()?; Ok(OpTy { op, layout, align: Some(layout.align.abi) }) } @@ -598,15 +559,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { layout: Option<TyAndLayout<'tcx>>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { match c.kind() { - ty::ConstKind::Param(_) | ty::ConstKind::Bound(..) => throw_inval!(TooGeneric), + ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(..) => throw_inval!(TooGeneric), ty::ConstKind::Error(DelaySpanBugEmitted { reported, .. }) => { throw_inval!(AlreadyReported(reported)) } ty::ConstKind::Unevaluated(uv) => { + // NOTE: We evaluate to a `ValTree` here as a check to ensure + // we're working with valid constants, even though we never need it. let instance = self.resolve(uv.def, uv.substs)?; - Ok(self.eval_to_allocation(GlobalId { instance, promoted: uv.promoted })?.into()) + let cid = GlobalId { instance, promoted: None }; + let _valtree = self + .tcx + .eval_to_valtree(self.param_env.and(cid))? + .unwrap_or_else(|| bug!("unable to create ValTree for {:?}", uv)); + + Ok(self.eval_to_allocation(cid)?.into()) } - ty::ConstKind::Infer(..) | ty::ConstKind::Placeholder(..) => { + ty::ConstKind::Bound(..) | ty::ConstKind::Infer(..) => { span_bug!(self.cur_span(), "const_to_op: Unexpected ConstKind {:?}", c) } ty::ConstKind::Value(valtree) => { @@ -622,9 +591,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { val: &mir::ConstantKind<'tcx>, layout: Option<TyAndLayout<'tcx>>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + // FIXME(const_prop): normalization needed b/c const prop lint in + // `mir_drops_elaborated_and_const_checked`, which happens before + // optimized MIR. Only after optimizing the MIR can we guarantee + // that the `RevealAll` pass has happened and that the body's consts + // are normalized, so any call to resolve before that needs to be + // manually normalized. + let val = self.tcx.normalize_erasing_regions(self.param_env, *val); match val { - mir::ConstantKind::Ty(ct) => self.const_to_op(*ct, layout), - mir::ConstantKind::Val(val, ty) => self.const_val_to_op(*val, *ty, layout), + mir::ConstantKind::Ty(ct) => self.const_to_op(ct, layout), + mir::ConstantKind::Val(val, ty) => self.const_val_to_op(val, ty, layout), + mir::ConstantKind::Unevaluated(uv, _) => { + let instance = self.resolve(uv.def, uv.substs)?; + Ok(self.eval_to_allocation(GlobalId { instance, promoted: uv.promoted })?.into()) + } } } @@ -727,7 +707,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Figure out which discriminant and variant this corresponds to. Ok(match *tag_encoding { TagEncoding::Direct => { - let scalar = tag_val.to_scalar()?; + let scalar = tag_val.to_scalar(); // Generate a specific error if `tag_val` is not an integer. // (`tag_bits` itself is only used for error messages below.) let tag_bits = scalar @@ -757,8 +737,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Return the cast value, and the index. (discr_val, index.0) } - TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => { - let tag_val = tag_val.to_scalar()?; + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { + let tag_val = tag_val.to_scalar(); // Compute the variant this niche value/"tag" corresponds to. With niche layout, // discriminant (encoded in niche/tag) and variant index are the same. let variants_start = niche_variants.start().as_u32(); @@ -775,7 +755,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { if !ptr_valid { throw_ub!(InvalidTag(dbg_val)) } - dataful_variant + untagged_variant } Ok(tag_bits) => { let tag_bits = tag_bits.assert_bits(tag_layout.size); @@ -785,9 +765,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let niche_start_val = ImmTy::from_uint(niche_start, tag_layout); let variant_index_relative_val = self.binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?; - let variant_index_relative = variant_index_relative_val - .to_scalar()? - .assert_bits(tag_val.layout.size); + let variant_index_relative = + variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size); // Check if this is in the range that indicates an actual discriminant. if variant_index_relative <= u128::from(variants_end - variants_start) { let variant_index_relative = u32::try_from(variant_index_relative) @@ -806,7 +785,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert!(usize::try_from(variant_index).unwrap() < variants_len); VariantIdx::from_u32(variant_index) } else { - dataful_variant + untagged_variant } } }; @@ -820,12 +799,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] +#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))] mod size_asserts { use super::*; + use rustc_data_structures::static_assert_size; // These are in alphabetical order, which is easy to maintain. - rustc_data_structures::static_assert_size!(Immediate, 56); - rustc_data_structures::static_assert_size!(ImmTy<'_>, 72); - rustc_data_structures::static_assert_size!(Operand, 64); - rustc_data_structures::static_assert_size!(OpTy<'_>, 88); + static_assert_size!(Immediate, 48); + static_assert_size!(ImmTy<'_>, 64); + static_assert_size!(Operand, 56); + static_assert_size!(OpTy<'_>, 80); } diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index f9912d706..1f1d06651 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -329,21 +329,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { match left.layout.ty.kind() { ty::Char => { assert_eq!(left.layout.ty, right.layout.ty); - let left = left.to_scalar()?; - let right = right.to_scalar()?; + let left = left.to_scalar(); + let right = right.to_scalar(); Ok(self.binary_char_op(bin_op, left.to_char()?, right.to_char()?)) } ty::Bool => { assert_eq!(left.layout.ty, right.layout.ty); - let left = left.to_scalar()?; - let right = right.to_scalar()?; + let left = left.to_scalar(); + let right = right.to_scalar(); Ok(self.binary_bool_op(bin_op, left.to_bool()?, right.to_bool()?)) } ty::Float(fty) => { assert_eq!(left.layout.ty, right.layout.ty); let ty = left.layout.ty; - let left = left.to_scalar()?; - let right = right.to_scalar()?; + let left = left.to_scalar(); + let right = right.to_scalar(); Ok(match fty { FloatTy::F32 => { self.binary_float_op(bin_op, ty, left.to_f32()?, right.to_f32()?) @@ -363,8 +363,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { right.layout.ty ); - let l = left.to_scalar()?.to_bits(left.layout.size)?; - let r = right.to_scalar()?.to_bits(right.layout.size)?; + let l = left.to_scalar().to_bits(left.layout.size)?; + let r = right.to_scalar().to_bits(right.layout.size)?; self.binary_int_op(bin_op, l, left.layout, r, right.layout) } _ if left.layout.ty.is_any_ptr() => { @@ -410,7 +410,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { use rustc_middle::mir::UnOp::*; let layout = val.layout; - let val = val.to_scalar()?; + let val = val.to_scalar(); trace!("Running unary op {:?}: {:?} ({:?})", un_op, val, layout.ty); match layout.ty.kind() { diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index f4571a1ca..b32889290 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -2,8 +2,6 @@ //! into a place. //! All high-level functions to write to memory work on places as destinations. -use std::hash::Hash; - use rustc_ast::Mutability; use rustc_middle::mir; use rustc_middle::ty; @@ -13,7 +11,7 @@ use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding, Vari use super::{ alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg, ConstAlloc, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, OpTy, Operand, - Pointer, Provenance, Scalar, ScalarMaybeUninit, + Pointer, Provenance, Scalar, }; #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] @@ -254,8 +252,6 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { // These are defined here because they produce a place. impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { #[inline(always)] - /// Note: do not call `as_ref` on the resulting place. This function should only be used to - /// read from the resulting mplace, not to get its address back. pub fn try_as_mplace(&self) -> Result<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> { match **self { Operand::Indirect(mplace) => { @@ -267,8 +263,6 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { #[inline(always)] #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - /// Note: do not call `as_ref` on the resulting place. This function should only be used to - /// read from the resulting mplace, not to get its address back. pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> { self.try_as_mplace().unwrap() } @@ -294,7 +288,7 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> where - Prov: Provenance + Eq + Hash + 'static, + Prov: Provenance + 'static, M: Machine<'mir, 'tcx, Provenance = Prov>, { /// Take a value, which represents a (thin or wide) reference, and make it a place. @@ -312,7 +306,7 @@ where let layout = self.layout_of(pointee_type)?; let (ptr, meta) = match **val { Immediate::Scalar(ptr) => (ptr, MemPlaceMeta::None), - Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta.check_init()?)), + Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta)), Immediate::Uninit => throw_ub!(InvalidUninitBytes(None)), }; @@ -467,7 +461,7 @@ where #[inline(always)] pub fn write_scalar( &mut self, - val: impl Into<ScalarMaybeUninit<M::Provenance>>, + val: impl Into<Scalar<M::Provenance>>, dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { self.write_immediate(Immediate::Scalar(val.into()), dest) @@ -644,9 +638,9 @@ where // Let us see if the layout is simple so we take a shortcut, // avoid force_allocation. - let src = match self.read_immediate_raw(src, /*force*/ false)? { + let src = match self.read_immediate_raw(src)? { Ok(src_val) => { - assert!(!src.layout.is_unsized(), "cannot have unsized immediates"); + assert!(!src.layout.is_unsized(), "cannot copy unsized immediates"); assert!( !dest.layout.is_unsized(), "the src is sized, so the dest must also be sized" @@ -823,7 +817,7 @@ where } abi::Variants::Multiple { tag_encoding: - TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start }, + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, tag: tag_layout, tag_field, .. @@ -831,7 +825,7 @@ where // No need to validate that the discriminant here because the // `TyAndLayout::for_variant()` call earlier already checks the variant is valid. - if variant_index != dataful_variant { + if variant_index != untagged_variant { let variants_start = niche_variants.start().as_u32(); let variant_index_relative = variant_index .as_u32() @@ -891,10 +885,13 @@ where #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] mod size_asserts { use super::*; + use rustc_data_structures::static_assert_size; // These are in alphabetical order, which is easy to maintain. - rustc_data_structures::static_assert_size!(MemPlaceMeta, 24); - rustc_data_structures::static_assert_size!(MemPlace, 40); - rustc_data_structures::static_assert_size!(MPlaceTy<'_>, 64); - rustc_data_structures::static_assert_size!(Place, 48); - rustc_data_structures::static_assert_size!(PlaceTy<'_>, 72); + static_assert_size!(MemPlaceMeta, 24); + static_assert_size!(MemPlace, 40); + static_assert_size!(MPlaceTy<'_>, 64); + #[cfg(not(bootstrap))] + static_assert_size!(Place, 40); + #[cfg(not(bootstrap))] + static_assert_size!(PlaceTy<'_>, 64); } diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index 742339f2b..77da8f104 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -1,14 +1,12 @@ //! This file implements "place projections"; basically a symmetric API for 3 types: MPlaceTy, OpTy, PlaceTy. //! -//! OpTy and PlaceTy genrally work by "let's see if we are actually an MPlaceTy, and do something custom if not". +//! OpTy and PlaceTy generally work by "let's see if we are actually an MPlaceTy, and do something custom if not". //! For PlaceTy, the custom thing is basically always to call `force_allocation` and then use the MPlaceTy logic anyway. //! For OpTy, the custom thing on field pojections has to be pretty clever (since `Operand::Immediate` can have fields), //! but for array/slice operations it only has to worry about `Operand::Uninit`. That makes the value part trivial, //! but we still need to do bounds checking and adjust the layout. To not duplicate that with MPlaceTy, we actually //! implement the logic on OpTy, and MPlaceTy calls that. -use std::hash::Hash; - use rustc_middle::mir; use rustc_middle::ty; use rustc_middle::ty::layout::LayoutOf; @@ -22,7 +20,7 @@ use super::{ // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> where - Prov: Provenance + Eq + Hash + 'static, + Prov: Provenance + 'static, M: Machine<'mir, 'tcx, Provenance = Prov>, { //# Field access @@ -100,6 +98,8 @@ where // This makes several assumptions about what layouts we will encounter; we match what // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`). let field_val: Immediate<_> = match (*base, base.layout.abi) { + // if the entire value is uninit, then so is the field (can happen in ConstProp) + (Immediate::Uninit, _) => Immediate::Uninit, // the field contains no information, can be left uninit _ if field_layout.is_zst() => Immediate::Uninit, // the field covers the entire type @@ -124,6 +124,7 @@ where b_val }) } + // everything else is a bug _ => span_bug!( self.cur_span(), "invalid field access on immediate {}, layout {:#?}", diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index fea158a9f..c6e04cbfb 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -53,7 +53,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.pop_stack_frame(/* unwinding */ true)?; return Ok(true); }; - let basic_block = &self.body().basic_blocks()[loc.block]; + let basic_block = &self.body().basic_blocks[loc.block]; if let Some(stmt) = basic_block.statements.get(loc.statement_index) { let old_frames = self.frame_idx(); @@ -114,13 +114,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { M::retag(self, *kind, &dest)?; } - // Call CopyNonOverlapping - CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => { - let src = self.eval_operand(src, None)?; - let dst = self.eval_operand(dst, None)?; - let count = self.eval_operand(count, None)?; - self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?; - } + Intrinsic(box ref intrinsic) => self.emulate_nondiverging_intrinsic(intrinsic)?, // Statements we do not track. AscribeUserType(..) => {} @@ -251,8 +245,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Len(place) => { let src = self.eval_place(place)?; - let mplace = self.force_allocation(&src)?; - let len = mplace.len(self)?; + let op = self.place_to_op(&src)?; + let len = op.len(self)?; self.write_scalar(Scalar::from_machine_usize(len, self), &dest)?; } diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index d563e35f9..50a82aa0e 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use rustc_ast::ast::InlineAsmOptions; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::Instance; use rustc_middle::{ @@ -129,8 +130,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } Assert { ref cond, expected, ref msg, target, cleanup } => { - let cond_val = - self.read_immediate(&self.eval_operand(cond, None)?)?.to_scalar()?.to_bool()?; + let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?; if expected == cond_val { self.go_to_block(target); } else { @@ -167,8 +167,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { terminator.kind ), - // Inline assembly can't be interpreted. - InlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"), + InlineAsm { template, ref operands, options, destination, .. } => { + M::eval_inline_asm(self, template, operands, options)?; + if options.contains(InlineAsmOptions::NORETURN) { + throw_ub_format!("returned from noreturn inline assembly"); + } + self.go_to_block( + destination + .expect("InlineAsm terminators without noreturn must have a destination"), + ) + } } Ok(()) @@ -215,12 +223,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => false, } }; - // Padding must be fully equal. - let pad_compat = || caller_abi.pad == callee_abi.pad; // When comparing the PassMode, we have to be smart about comparing the attributes. - let arg_attr_compat = |a1: ArgAttributes, a2: ArgAttributes| { + let arg_attr_compat = |a1: &ArgAttributes, a2: &ArgAttributes| { // There's only one regular attribute that matters for the call ABI: InReg. - // Everything else is things like noalias, dereferencable, nonnull, ... + // Everything else is things like noalias, dereferenceable, nonnull, ... // (This also applies to pointee_size, pointee_align.) if a1.regular.contains(ArgAttribute::InReg) != a2.regular.contains(ArgAttribute::InReg) { @@ -233,13 +239,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } return true; }; - let mode_compat = || match (caller_abi.mode, callee_abi.mode) { + let mode_compat = || match (&caller_abi.mode, &callee_abi.mode) { (PassMode::Ignore, PassMode::Ignore) => true, (PassMode::Direct(a1), PassMode::Direct(a2)) => arg_attr_compat(a1, a2), (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => { arg_attr_compat(a1, a2) && arg_attr_compat(b1, b2) } - (PassMode::Cast(c1), PassMode::Cast(c2)) => c1 == c2, + (PassMode::Cast(c1, pad1), PassMode::Cast(c2, pad2)) => c1 == c2 && pad1 == pad2, ( PassMode::Indirect { attrs: a1, extra_attrs: None, on_stack: s1 }, PassMode::Indirect { attrs: a2, extra_attrs: None, on_stack: s2 }, @@ -251,7 +257,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => false, }; - if layout_compat() && pad_compat() && mode_compat() { + if layout_compat() && mode_compat() { return true; } trace!( @@ -534,7 +540,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let mut non_zst_field = None; for i in 0..receiver.layout.fields.count() { let field = self.operand_field(&receiver, i)?; - if !field.layout.is_zst() { + let zst = + field.layout.is_zst() && field.layout.align.abi.bytes() == 1; + if !zst { assert!( non_zst_field.is_none(), "multiple non-ZST fields in dyn receiver type {}", @@ -557,7 +565,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .tcx .struct_tail_erasing_lifetimes(receiver_place.layout.ty, self.param_env); let ty::Dynamic(data, ..) = receiver_tail.kind() else { - span_bug!(self.cur_span(), "dyanmic call on non-`dyn` type {}", receiver_tail) + span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail) }; // Get the required information from the vtable. diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs index b3a511d5a..cab23b724 100644 --- a/compiler/rustc_const_eval/src/interpret/traits.rs +++ b/compiler/rustc_const_eval/src/interpret/traits.rs @@ -32,7 +32,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Ok(vtable_ptr.into()) } - /// Returns a high-level representation of the entires of the given vtable. + /// Returns a high-level representation of the entries of the given vtable. pub fn get_vtable_entries( &self, vtable: Pointer<Option<M::Provenance>>, diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 0e50d1ed4..14aaee6ac 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -5,9 +5,10 @@ //! to be const-safe. use std::convert::TryFrom; -use std::fmt::Write; +use std::fmt::{Display, Write}; use std::num::NonZeroUsize; +use rustc_ast::Mutability; use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; use rustc_middle::mir::interpret::InterpError; @@ -19,9 +20,11 @@ use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, VariantIdx, Variants, Wr use std::hash::Hash; +// for the validation errors +use super::UndefinedBehaviorInfo::*; use super::{ - alloc_range, CheckInAllocMsg, GlobalAlloc, Immediate, InterpCx, InterpResult, MPlaceTy, - Machine, MemPlaceMeta, OpTy, Scalar, ScalarMaybeUninit, ValueVisitor, + CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, + MemPlaceMeta, OpTy, Scalar, ValueVisitor, }; macro_rules! throw_validation_failure { @@ -59,6 +62,7 @@ macro_rules! throw_validation_failure { /// }); /// ``` /// +/// The patterns must be of type `UndefinedBehaviorInfo`. /// An additional expected parameter can also be added to the failure message: /// /// ``` @@ -86,7 +90,7 @@ macro_rules! try_validation { // allocation here as this can only slow down builds that fail anyway. Err(e) => match e.kind() { $( - $($p)|+ => + InterpError::UndefinedBehavior($($p)|+) => throw_validation_failure!( $where, { $( $what_fmt ),+ } $( expected { $( $expected_fmt ),+ } )? @@ -304,6 +308,26 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' Ok(r) } + fn read_immediate( + &self, + op: &OpTy<'tcx, M::Provenance>, + expected: impl Display, + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { + Ok(try_validation!( + self.ecx.read_immediate(op), + self.path, + InvalidUninitBytes(None) => { "uninitialized memory" } expected { "{expected}" } + )) + } + + fn read_scalar( + &self, + op: &OpTy<'tcx, M::Provenance>, + expected: impl Display, + ) -> InterpResult<'tcx, Scalar<M::Provenance>> { + Ok(self.read_immediate(op, expected)?.to_scalar()) + } + fn check_wide_ptr_meta( &mut self, meta: MemPlaceMeta<M::Provenance>, @@ -317,8 +341,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let (_ty, _trait) = try_validation!( self.ecx.get_ptr_vtable(vtable), self.path, - err_ub!(DanglingIntPointer(..)) | - err_ub!(InvalidVTablePointer(..)) => + DanglingIntPointer(..) | + InvalidVTablePointer(..) => { "{vtable}" } expected { "a vtable pointer" }, ); // FIXME: check if the type/trait match what ty::Dynamic says? @@ -344,14 +368,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' value: &OpTy<'tcx, M::Provenance>, kind: &str, ) -> InterpResult<'tcx> { - let value = self.ecx.read_immediate(value)?; + let place = + self.ecx.ref_to_mplace(&self.read_immediate(value, format_args!("a {kind}"))?)?; // Handle wide pointers. // Check metadata early, for better diagnostics - let place = try_validation!( - self.ecx.ref_to_mplace(&value), - self.path, - err_ub!(InvalidUninitBytes(None)) => { "uninitialized {}", kind }, - ); if place.layout.is_unsized() { self.check_wide_ptr_meta(place.meta, place.layout)?; } @@ -359,7 +379,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let size_and_align = try_validation!( self.ecx.size_and_align_of_mplace(&place), self.path, - err_ub!(InvalidMeta(msg)) => { "invalid {} metadata: {}", kind, msg }, + InvalidMeta(msg) => { "invalid {} metadata: {}", kind, msg }, ); let (size, align) = size_and_align // for the purpose of validity, consider foreign types to have @@ -375,21 +395,21 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message ), self.path, - err_ub!(AlignmentCheckFailed { required, has }) => + AlignmentCheckFailed { required, has } => { "an unaligned {kind} (required {} byte alignment but found {})", required.bytes(), has.bytes() }, - err_ub!(DanglingIntPointer(0, _)) => + DanglingIntPointer(0, _) => { "a null {kind}" }, - err_ub!(DanglingIntPointer(i, _)) => + DanglingIntPointer(i, _) => { "a dangling {kind} (address {i:#x} is unallocated)" }, - err_ub!(PointerOutOfBounds { .. }) => + PointerOutOfBounds { .. } => { "a dangling {kind} (going beyond the bounds of its allocation)" }, // This cannot happen during const-eval (because interning already detects // dangling pointers), but it can happen in Miri. - err_ub!(PointerUseAfterFree(..)) => + PointerUseAfterFree(..) => { "a dangling {kind} (use-after-free)" }, ); // Do not allow pointers to uninhabited types. @@ -403,34 +423,51 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // Proceed recursively even for ZST, no reason to skip them! // `!` is a ZST and we want to validate it. if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr) { - // Special handling for pointers to statics (irrespective of their type). + // Let's see what kind of memory this points to. let alloc_kind = self.ecx.tcx.try_get_global_alloc(alloc_id); - if let Some(GlobalAlloc::Static(did)) = alloc_kind { - assert!(!self.ecx.tcx.is_thread_local_static(did)); - assert!(self.ecx.tcx.is_static(did)); - if matches!( - self.ctfe_mode, - Some(CtfeValidationMode::Const { allow_static_ptrs: false, .. }) - ) { - // See const_eval::machine::MemoryExtra::can_access_statics for why - // this check is so important. - // This check is reachable when the const just referenced the static, - // but never read it (so we never entered `before_access_global`). - throw_validation_failure!(self.path, - { "a {} pointing to a static variable", kind } - ); + match alloc_kind { + Some(GlobalAlloc::Static(did)) => { + // Special handling for pointers to statics (irrespective of their type). + assert!(!self.ecx.tcx.is_thread_local_static(did)); + assert!(self.ecx.tcx.is_static(did)); + if matches!( + self.ctfe_mode, + Some(CtfeValidationMode::Const { allow_static_ptrs: false, .. }) + ) { + // See const_eval::machine::MemoryExtra::can_access_statics for why + // this check is so important. + // This check is reachable when the const just referenced the static, + // but never read it (so we never entered `before_access_global`). + throw_validation_failure!(self.path, + { "a {} pointing to a static variable in a constant", kind } + ); + } + // We skip recursively checking other statics. These statics must be sound by + // themselves, and the only way to get broken statics here is by using + // unsafe code. + // The reasons we don't check other statics is twofold. For one, in all + // sound cases, the static was already validated on its own, and second, we + // trigger cycle errors if we try to compute the value of the other static + // and that static refers back to us. + // We might miss const-invalid data, + // but things are still sound otherwise (in particular re: consts + // referring to statics). + return Ok(()); } - // We skip checking other statics. These statics must be sound by - // themselves, and the only way to get broken statics here is by using - // unsafe code. - // The reasons we don't check other statics is twofold. For one, in all - // sound cases, the static was already validated on its own, and second, we - // trigger cycle errors if we try to compute the value of the other static - // and that static refers back to us. - // We might miss const-invalid data, - // but things are still sound otherwise (in particular re: consts - // referring to statics). - return Ok(()); + Some(GlobalAlloc::Memory(alloc)) => { + if alloc.inner().mutability == Mutability::Mut + && matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. })) + { + // This should be unreachable, but if someone manages to copy a pointer + // out of a `static`, then that pointer might point to mutable memory, + // and we would catch that here. + throw_validation_failure!(self.path, + { "a {} pointing to mutable memory in a constant", kind } + ); + } + } + // Nothing to check for these. + None | Some(GlobalAlloc::Function(..) | GlobalAlloc::VTable(..)) => {} } } let path = &self.path; @@ -446,20 +483,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' Ok(()) } - fn read_scalar( - &self, - op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, ScalarMaybeUninit<M::Provenance>> { - self.ecx.read_scalar(op) - } - - fn read_immediate_forced( - &self, - op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, Immediate<M::Provenance>> { - Ok(*self.ecx.read_immediate_raw(op, /*force*/ true)?.unwrap()) - } - /// Check if this is a value of primitive type, and if yes check the validity of the value /// at that type. Return `true` if the type is indeed primitive. fn try_visit_primitive( @@ -470,41 +493,39 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let ty = value.layout.ty; match ty.kind() { ty::Bool => { - let value = self.read_scalar(value)?; + let value = self.read_scalar(value, "a boolean")?; try_validation!( value.to_bool(), self.path, - err_ub!(InvalidBool(..)) | err_ub!(InvalidUninitBytes(None)) => + InvalidBool(..) => { "{:x}", value } expected { "a boolean" }, ); Ok(true) } ty::Char => { - let value = self.read_scalar(value)?; + let value = self.read_scalar(value, "a unicode scalar value")?; try_validation!( value.to_char(), self.path, - err_ub!(InvalidChar(..)) | err_ub!(InvalidUninitBytes(None)) => + InvalidChar(..) => { "{:x}", value } expected { "a valid unicode scalar value (in `0..=0x10FFFF` but not in `0xD800..=0xDFFF`)" }, ); Ok(true) } ty::Float(_) | ty::Int(_) | ty::Uint(_) => { - let value = self.read_scalar(value)?; // NOTE: Keep this in sync with the array optimization for int/float // types below! - if M::enforce_number_init(self.ecx) { - try_validation!( - value.check_init(), - self.path, - err_ub!(InvalidUninitBytes(..)) => - { "{:x}", value } expected { "initialized bytes" } - ); - } + let value = self.read_scalar( + value, + if matches!(ty.kind(), ty::Float(..)) { + "a floating point number" + } else { + "an integer" + }, + )?; // As a special exception we *do* match on a `Scalar` here, since we truly want // to know its underlying representation (and *not* cast it to an integer). - let is_ptr = value.check_init().map_or(false, |v| matches!(v, Scalar::Ptr(..))); - if is_ptr { + if matches!(value, Scalar::Ptr(..)) { throw_validation_failure!(self.path, { "{:x}", value } expected { "plain (non-pointer) bytes" } ) @@ -515,11 +536,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // We are conservative with uninit for integers, but try to // actually enforce the strict rules for raw pointers (mostly because // that lets us re-use `ref_to_mplace`). - let place = try_validation!( - self.ecx.read_immediate(value).and_then(|ref i| self.ecx.ref_to_mplace(i)), - self.path, - err_ub!(InvalidUninitBytes(None)) => { "uninitialized raw pointer" }, - ); + let place = + self.ecx.ref_to_mplace(&self.read_immediate(value, "a raw pointer")?)?; if place.layout.is_unsized() { self.check_wide_ptr_meta(place.meta, place.layout)?; } @@ -527,7 +545,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' } ty::Ref(_, ty, mutbl) => { if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. })) - && *mutbl == hir::Mutability::Mut + && *mutbl == Mutability::Mut { // A mutable reference inside a const? That does not seem right (except if it is // a ZST). @@ -540,11 +558,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' Ok(true) } ty::FnPtr(_sig) => { - let value = try_validation!( - self.ecx.read_scalar(value).and_then(|v| v.check_init()), - self.path, - err_ub!(InvalidUninitBytes(None)) => { "uninitialized bytes" } expected { "a proper pointer or integer value" }, - ); + let value = self.read_scalar(value, "a function pointer")?; // If we check references recursively, also check that this points to a function. if let Some(_) = self.ref_tracking { @@ -552,8 +566,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let _fn = try_validation!( self.ecx.get_ptr_fn(ptr), self.path, - err_ub!(DanglingIntPointer(..)) | - err_ub!(InvalidFunctionPointer(..)) => + DanglingIntPointer(..) | + InvalidFunctionPointer(..) => { "{ptr}" } expected { "a function pointer" }, ); // FIXME: Check if the signature matches @@ -595,40 +609,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' fn visit_scalar( &mut self, - scalar: ScalarMaybeUninit<M::Provenance>, + scalar: Scalar<M::Provenance>, scalar_layout: ScalarAbi, ) -> InterpResult<'tcx> { - // We check `is_full_range` in a slightly complicated way because *if* we are checking - // number validity, then we want to ensure that `Scalar::Initialized` is indeed initialized, - // i.e. that we go over the `check_init` below. let size = scalar_layout.size(self.ecx); - let is_full_range = match scalar_layout { - ScalarAbi::Initialized { .. } => { - if M::enforce_number_init(self.ecx) { - false // not "full" since uninit is not accepted - } else { - scalar_layout.is_always_valid(self.ecx) - } - } - ScalarAbi::Union { .. } => true, - }; - if is_full_range { - // Nothing to check. Cruciall we don't even `read_scalar` until here, since that would - // fail for `Union` scalars! - return Ok(()); - } - // We have something to check: it must at least be initialized. let valid_range = scalar_layout.valid_range(self.ecx); let WrappingRange { start, end } = valid_range; let max_value = size.unsigned_int_max(); assert!(end <= max_value); - let value = try_validation!( - scalar.check_init(), - self.path, - err_ub!(InvalidUninitBytes(None)) => { "{:x}", scalar } - expected { "something {}", wrapping_range_format(valid_range, max_value) }, - ); - let bits = match value.try_to_int() { + let bits = match scalar.try_to_int() { Ok(int) => int.assert_bits(size), Err(_) => { // So this is a pointer then, and casting to an int failed. @@ -636,7 +625,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // We support 2 kinds of ranges here: full range, and excluding zero. if start == 1 && end == max_value { // Only null is the niche. So make sure the ptr is NOT null. - if self.ecx.scalar_may_be_null(value)? { + if self.ecx.scalar_may_be_null(scalar)? { throw_validation_failure!(self.path, { "a potentially null pointer" } expected { @@ -693,9 +682,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> Ok(try_validation!( this.ecx.read_discriminant(op), this.path, - err_ub!(InvalidTag(val)) => + InvalidTag(val) => { "{:x}", val } expected { "a valid enum tag" }, - err_ub!(InvalidUninitBytes(None)) => + InvalidUninitBytes(None) => { "uninitialized bytes" } expected { "a valid enum tag" }, ) .1) @@ -788,10 +777,11 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> ); } Abi::Scalar(scalar_layout) => { - // We use a 'forced' read because we always need a `Immediate` here - // and treating "partially uninit" as "fully uninit" is fine for us. - let scalar = self.read_immediate_forced(op)?.to_scalar_or_uninit(); - self.visit_scalar(scalar, scalar_layout)?; + if !scalar_layout.is_uninit_valid() { + // There is something to check here. + let scalar = self.read_scalar(op, "initiailized scalar value")?; + self.visit_scalar(scalar, scalar_layout)?; + } } Abi::ScalarPair(a_layout, b_layout) => { // There is no `rustc_layout_scalar_valid_range_start` for pairs, so @@ -799,10 +789,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // but that can miss bugs in layout computation. Layout computation // is subtle due to enums having ScalarPair layout, where one field // is the discriminant. - if cfg!(debug_assertions) { - // We use a 'forced' read because we always need a `Immediate` here - // and treating "partially uninit" as "fully uninit" is fine for us. - let (a, b) = self.read_immediate_forced(op)?.to_scalar_or_uninit_pair(); + if cfg!(debug_assertions) + && !a_layout.is_uninit_valid() + && !b_layout.is_uninit_valid() + { + // We can only proceed if *both* scalars need to be initialized. + // FIXME: find a way to also check ScalarPair when one side can be uninit but + // the other must be init. + let (a, b) = + self.read_immediate(op, "initiailized scalar value")?.to_scalar_pair(); self.visit_scalar(a, a_layout)?; self.visit_scalar(b, b_layout)?; } @@ -830,9 +825,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate let len = mplace.len(self.ecx)?; try_validation!( - self.ecx.read_bytes_ptr(mplace.ptr, Size::from_bytes(len)), + self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len)), self.path, - err_ub!(InvalidUninitBytes(..)) => { "uninitialized data in `str`" }, + InvalidUninitBytes(..) => { "uninitialized data in `str`" }, ); } ty::Array(tys, ..) | ty::Slice(tys) @@ -880,13 +875,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // We also accept uninit, for consistency with the slow path. let alloc = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)?.expect("we already excluded size 0"); - match alloc.check_bytes( - alloc_range(Size::ZERO, size), - /*allow_uninit*/ !M::enforce_number_init(self.ecx), - /*allow_ptr*/ false, - ) { + match alloc.get_bytes_strip_provenance() { // In the happy case, we needn't check anything else. - Ok(()) => {} + Ok(_) => {} // Some error happened, try to provide a more detailed description. Err(err) => { // For some errors we might be able to provide extra information. @@ -981,6 +972,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// It will error if the bits at the destination do not match the ones described by the layout. #[inline(always)] pub fn validate_operand(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx> { + // Note that we *could* actually be in CTFE here with `-Zextra-const-ub-checks`, but it's + // still correct to not use `ctfe_mode`: that mode is for validation of the final constant + // value, it rules out things like `UnsafeCell` in awkward places. It also can make checking + // recurse through references which, for now, we don't want here, either. self.validate_operand_internal(op, vec![], None, None) } } diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index 72ac6af68..9f47d302a 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -10,7 +10,7 @@ Rust MIR: a lowered representation of Rust. #![feature(decl_macro)] #![feature(exact_size_is_empty)] #![feature(let_chains)] -#![feature(let_else)] +#![cfg_attr(bootstrap, feature(let_else))] #![feature(map_try_insert)] #![feature(min_specialization)] #![feature(slice_ptr_get)] diff --git a/compiler/rustc_const_eval/src/might_permit_raw_init.rs b/compiler/rustc_const_eval/src/might_permit_raw_init.rs index f971c2238..37ffa19cc 100644 --- a/compiler/rustc_const_eval/src/might_permit_raw_init.rs +++ b/compiler/rustc_const_eval/src/might_permit_raw_init.rs @@ -13,7 +13,11 @@ pub fn might_permit_raw_init<'tcx>( let strict = tcx.sess.opts.unstable_opts.strict_init_checks; if strict { - let machine = CompileTimeInterpreter::new(Limit::new(0), false); + let machine = CompileTimeInterpreter::new( + Limit::new(0), + /*can_access_statics:*/ false, + /*check_alignment:*/ true, + ); let mut cx = InterpCx::new(tcx, rustc_span::DUMMY_SP, ParamEnv::reveal_all(), machine); diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs index 0adb88a18..7e15858c8 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs @@ -135,7 +135,7 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> { // qualifs for the return type. let return_block = ccx .body - .basic_blocks() + .basic_blocks .iter_enumerated() .find(|(_, block)| matches!(block.terminator().kind, TerminatorKind::Return)) .map(|(bb, _)| bb); @@ -546,6 +546,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { // Since no pointer can ever get exposed (rejected above), this is easy to support. } + Rvalue::Cast(CastKind::DynStar, _, _) => { + unimplemented!() + } + Rvalue::Cast(CastKind::Misc, _, _) => {} Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => {} @@ -678,7 +682,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { | StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs index 338022616..5fb4bf638 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs @@ -1,6 +1,7 @@ //! Concrete error types for all operations which may be invalid in a certain const context. use hir::def_id::LocalDefId; +use hir::ConstContext; use rustc_errors::{ error_code, struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, }; @@ -23,8 +24,11 @@ use rustc_trait_selection::traits::SelectionContext; use super::ConstCx; use crate::errors::{ - MutDerefErr, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr, - TransientMutBorrowErr, TransientMutBorrowErrRaw, + InteriorMutabilityBorrow, InteriorMutableDataRefer, MutDerefErr, NonConstFmtMacroCall, + NonConstFnCall, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr, + TransientMutBorrowErr, TransientMutBorrowErrRaw, UnallowedFnPointerCall, + UnallowedHeapAllocations, UnallowedInlineAsm, UnallowedMutableRefs, UnallowedMutableRefsRaw, + UnallowedOpInConstContext, UnstableConstFn, }; use crate::util::{call_kind, CallDesugaringKind, CallKind}; @@ -96,10 +100,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallIndirect { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - ccx.tcx.sess.struct_span_err( - span, - &format!("function pointer calls are not allowed in {}s", ccx.const_kind()), - ) + ccx.tcx.sess.create_err(UnallowedFnPointerCall { span, kind: ccx.const_kind() }) } } @@ -307,22 +308,13 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> { err } _ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::ArgumentV1Methods) => { - struct_span_err!( - ccx.tcx.sess, - span, - E0015, - "cannot call non-const formatting macro in {}s", - ccx.const_kind(), - ) + ccx.tcx.sess.create_err(NonConstFmtMacroCall { span, kind: ccx.const_kind() }) } - _ => struct_span_err!( - ccx.tcx.sess, + _ => ccx.tcx.sess.create_err(NonConstFnCall { span, - E0015, - "cannot call non-const fn `{}` in {}s", - ccx.tcx.def_path_str_with_substs(callee, substs), - ccx.const_kind(), - ), + def_path_str: ccx.tcx.def_path_str_with_substs(callee, substs), + kind: ccx.const_kind(), + }), }; err.note(&format!( @@ -331,6 +323,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> { ccx.const_kind(), )); + if let ConstContext::Static(_) = ccx.const_kind() { + err.note("consider wrapping this expression in `Lazy::new(|| ...)` from the `once_cell` crate: https://crates.io/crates/once_cell"); + } + err } } @@ -349,10 +345,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallUnstable { ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { let FnCallUnstable(def_id, feature) = *self; - let mut err = ccx.tcx.sess.struct_span_err( - span, - &format!("`{}` is not yet stable as a const fn", ccx.tcx.def_path_str(def_id)), - ); + let mut err = ccx + .tcx + .sess + .create_err(UnstableConstFn { span, def_path: ccx.tcx.def_path_str(def_id) }); if ccx.is_const_stable_const_fn() { err.help("const-stable functions can only call other const-stable functions"); @@ -387,9 +383,12 @@ impl<'tcx> NonConstOp<'tcx> for Generator { ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { let msg = format!("{}s are not allowed in {}s", self.0, ccx.const_kind()); if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 { - feature_err(&ccx.tcx.sess.parse_sess, sym::const_async_blocks, span, &msg) + ccx.tcx.sess.create_feature_err( + UnallowedOpInConstContext { span, msg }, + sym::const_async_blocks, + ) } else { - ccx.tcx.sess.struct_span_err(span, &msg) + ccx.tcx.sess.create_err(UnallowedOpInConstContext { span, msg }) } } } @@ -402,23 +401,11 @@ impl<'tcx> NonConstOp<'tcx> for HeapAllocation { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - let mut err = struct_span_err!( - ccx.tcx.sess, + ccx.tcx.sess.create_err(UnallowedHeapAllocations { span, - E0010, - "allocations are not allowed in {}s", - ccx.const_kind() - ); - err.span_label(span, format!("allocation not allowed in {}s", ccx.const_kind())); - if ccx.tcx.sess.teach(&err.get_code().unwrap()) { - err.note( - "The value of statics and constants must be known at compile time, \ - and they live for the entire lifetime of a program. Creating a boxed \ - value allocates memory on the heap at runtime, and therefore cannot \ - be done at compile time.", - ); - } - err + kind: ccx.const_kind(), + teach: ccx.tcx.sess.teach(&error_code!(E0010)).then_some(()), + }) } } @@ -430,13 +417,7 @@ impl<'tcx> NonConstOp<'tcx> for InlineAsm { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - struct_span_err!( - ccx.tcx.sess, - span, - E0015, - "inline assembly is not allowed in {}s", - ccx.const_kind() - ) + ccx.tcx.sess.create_err(UnallowedInlineAsm { span, kind: ccx.const_kind() }) } } @@ -482,12 +463,7 @@ impl<'tcx> NonConstOp<'tcx> for TransientCellBorrow { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - feature_err( - &ccx.tcx.sess.parse_sess, - sym::const_refs_to_cell, - span, - "cannot borrow here, since the borrowed element may contain interior mutability", - ) + ccx.tcx.sess.create_feature_err(InteriorMutabilityBorrow { span }, sym::const_refs_to_cell) } } @@ -502,32 +478,22 @@ impl<'tcx> NonConstOp<'tcx> for CellBorrow { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - let mut err = struct_span_err!( - ccx.tcx.sess, - span, - E0492, - "{}s cannot refer to interior mutable data", - ccx.const_kind(), - ); - err.span_label( - span, - "this borrow of an interior mutable value may end up in the final value", - ); + // FIXME: Maybe a more elegant solution to this if else case if let hir::ConstContext::Static(_) = ccx.const_kind() { - err.help( - "to fix this, the value can be extracted to a separate \ - `static` item and then referenced", - ); - } - if ccx.tcx.sess.teach(&err.get_code().unwrap()) { - err.note( - "A constant containing interior mutable data behind a reference can allow you - to modify that data. This would make multiple uses of a constant to be able to - see different values and allow circumventing the `Send` and `Sync` requirements - for shared mutable data, which is unsound.", - ); + ccx.tcx.sess.create_err(InteriorMutableDataRefer { + span, + opt_help: Some(()), + kind: ccx.const_kind(), + teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()), + }) + } else { + ccx.tcx.sess.create_err(InteriorMutableDataRefer { + span, + opt_help: None, + kind: ccx.const_kind(), + teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()), + }) } - err } } @@ -553,33 +519,18 @@ impl<'tcx> NonConstOp<'tcx> for MutBorrow { ccx: &ConstCx<'_, 'tcx>, span: Span, ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> { - let raw = match self.0 { - hir::BorrowKind::Raw => "raw ", - hir::BorrowKind::Ref => "", - }; - - let mut err = struct_span_err!( - ccx.tcx.sess, - span, - E0764, - "{}mutable references are not allowed in the final value of {}s", - raw, - ccx.const_kind(), - ); - - if ccx.tcx.sess.teach(&err.get_code().unwrap()) { - err.note( - "References in statics and constants may only refer \ - to immutable values.\n\n\ - Statics are shared everywhere, and if they refer to \ - mutable data one might violate memory safety since \ - holding multiple mutable references to shared data \ - is not allowed.\n\n\ - If you really want global mutable state, try using \ - static mut or a global UnsafeCell.", - ); + match self.0 { + hir::BorrowKind::Raw => ccx.tcx.sess.create_err(UnallowedMutableRefsRaw { + span, + kind: ccx.const_kind(), + teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()), + }), + hir::BorrowKind::Ref => ccx.tcx.sess.create_err(UnallowedMutableRefs { + span, + kind: ccx.const_kind(), + teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()), + }), } - err } } diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs index c8a63c9c3..6c73ef5a8 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs @@ -5,12 +5,11 @@ use rustc_errors::ErrorGuaranteed; use rustc_hir::LangItem; use rustc_infer::infer::TyCtxtInferExt; -use rustc_infer::traits::TraitEngine; use rustc_middle::mir::*; use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty}; use rustc_span::DUMMY_SP; use rustc_trait_selection::traits::{ - self, ImplSource, Obligation, ObligationCause, SelectionContext, TraitEngineExt, + self, ImplSource, Obligation, ObligationCause, SelectionContext, }; use super::ConstCx; @@ -189,15 +188,8 @@ impl Qualif for NeedsNonConstDrop { return false; } - // If we successfully found one, then select all of the predicates - // implied by our const drop impl. - let mut fcx = <dyn TraitEngine<'tcx>>::new(cx.tcx); - for nested in impl_src.nested_obligations() { - fcx.register_predicate_obligation(&infcx, nested); - } - // If we had any errors, then it's bad - !fcx.select_all_or_error(&infcx).is_empty() + !traits::fully_solve_obligations(&infcx, impl_src.nested_obligations()).is_empty() }) } @@ -354,31 +346,43 @@ where }; // Check the qualifs of the value of `const` items. - if let Some(ct) = constant.literal.const_for_ty() { - if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted }) = ct.kind() - { - // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible - // only for `NeedsNonConstDrop` with precise drop checking. This is the only const - // check performed after the promotion. Verify that with an assertion. - assert!(promoted.is_none() || Q::ALLOW_PROMOTED); - // Don't peek inside trait associated constants. - if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() { - let qualifs = if let Some((did, param_did)) = def.as_const_arg() { - cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did)) - } else { - cx.tcx.at(constant.span).mir_const_qualif(def.did) - }; - - if !Q::in_qualifs(&qualifs) { - return false; - } + // FIXME(valtrees): check whether const qualifs should behave the same + // way for type and mir constants. + let uneval = match constant.literal { + ConstantKind::Ty(ct) if matches!(ct.kind(), ty::ConstKind::Unevaluated(_)) => { + let ty::ConstKind::Unevaluated(uv) = ct.kind() else { unreachable!() }; + + Some(uv.expand()) + } + ConstantKind::Ty(_) => None, + ConstantKind::Unevaluated(uv, _) => Some(uv), + ConstantKind::Val(..) => None, + }; - // Just in case the type is more specific than - // the definition, e.g., impl associated const - // with type parameters, take it into account. + if let Some(ty::Unevaluated { def, substs: _, promoted }) = uneval { + // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible + // only for `NeedsNonConstDrop` with precise drop checking. This is the only const + // check performed after the promotion. Verify that with an assertion. + assert!(promoted.is_none() || Q::ALLOW_PROMOTED); + + // Don't peek inside trait associated constants. + if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() { + let qualifs = if let Some((did, param_did)) = def.as_const_arg() { + cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did)) + } else { + cx.tcx.at(constant.span).mir_const_qualif(def.did) + }; + + if !Q::in_qualifs(&qualifs) { + return false; } + + // Just in case the type is more specific than + // the definition, e.g., impl associated const + // with type parameters, take it into account. } } + // Otherwise use the qualifs of the type. Q::in_any_value_of_ty(cx, constant.literal.ty()) } diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs index ed4d8c95d..f7a7cc88a 100644 --- a/compiler/rustc_const_eval/src/transform/promote_consts.rs +++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs @@ -41,7 +41,7 @@ pub struct PromoteTemps<'tcx> { impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> { fn phase_change(&self) -> Option<MirPhase> { - Some(MirPhase::ConstsPromoted) + Some(MirPhase::Analysis(AnalysisPhase::Initial)) } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -710,7 +710,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { } fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) { - let last = self.promoted.basic_blocks().last().unwrap(); + let last = self.promoted.basic_blocks.last().unwrap(); let data = &mut self.promoted[last]; data.statements.push(Statement { source_info: SourceInfo::outermost(span), @@ -803,7 +803,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { self.visit_operand(arg, loc); } - let last = self.promoted.basic_blocks().last().unwrap(); + let last = self.promoted.basic_blocks.last().unwrap(); let new_target = self.new_block(); *self.promoted[last].terminator_mut() = Terminator { @@ -839,27 +839,16 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { let mut promoted_operand = |ty, span| { promoted.span = span; promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span); - let _const = tcx.mk_const(ty::ConstS { - ty, - kind: ty::ConstKind::Unevaluated(ty::Unevaluated { - def, - substs: InternalSubsts::for_item(tcx, def.did, |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }), - promoted: Some(promoted_id), - }), - }); + let substs = tcx.erase_regions(InternalSubsts::identity_for_item(tcx, def.did)); + let uneval = ty::Unevaluated { def, substs, promoted: Some(promoted_id) }; Operand::Constant(Box::new(Constant { span, user_ty: None, - literal: ConstantKind::from_const(_const, tcx), + literal: ConstantKind::Unevaluated(uneval, ty), })) }; + let blocks = self.source.basic_blocks.as_mut(); let local_decls = &mut self.source.local_decls; let loc = candidate.location; @@ -969,7 +958,7 @@ pub fn promote_candidates<'tcx>( let mut scope = body.source_scopes[body.source_info(candidate.location).scope].clone(); scope.parent_scope = None; - let promoted = Body::new( + let mut promoted = Body::new( body.source, // `promoted` gets filled in below IndexVec::new(), IndexVec::from_elem_n(scope, 1), @@ -981,6 +970,7 @@ pub fn promote_candidates<'tcx>( body.generator_kind(), body.tainted_by_errors, ); + promoted.phase = MirPhase::Analysis(AnalysisPhase::Initial); let promoter = Promoter { promoted, @@ -1046,7 +1036,7 @@ pub fn is_const_fn_in_array_repeat_expression<'tcx>( _ => {} } - for block in body.basic_blocks() { + for block in body.basic_blocks.iter() { if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) = &block.terminator { diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 15e820f2d..4aa98cb13 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::NonUseContext::VarDebugInfo; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::{ - traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location, - MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, Rvalue, SourceScope, - Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK, + traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping, + Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef, + ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator, + TerminatorKind, UnOp, START_BLOCK, }; use rustc_middle::ty::fold::BottomUpFolder; use rustc_middle::ty::subst::Subst; @@ -89,9 +90,8 @@ pub fn equal_up_to_regions<'tcx>( // Normalize lifetimes away on both sides, then compare. let normalize = |ty: Ty<'tcx>| { - tcx.normalize_erasing_regions( - param_env, - ty.fold_with(&mut BottomUpFolder { + tcx.try_normalize_erasing_regions(param_env, ty).unwrap_or(ty).fold_with( + &mut BottomUpFolder { tcx, // FIXME: We erase all late-bound lifetimes, but this is not fully correct. // If you have a type like `<for<'a> fn(&'a u32) as SomeTrait>::Assoc`, @@ -103,7 +103,7 @@ pub fn equal_up_to_regions<'tcx>( // Leave consts and types unchanged. ct_op: |ct| ct, ty_op: |ty| ty, - }), + }, ) }; tcx.infer_ctxt().enter(|infcx| infcx.can_eq(param_env, normalize(src), normalize(dest)).is_ok()) @@ -142,8 +142,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { if bb == START_BLOCK { self.fail(location, "start block must not have predecessors") } - if let Some(bb) = self.body.basic_blocks().get(bb) { - let src = self.body.basic_blocks().get(location.block).unwrap(); + if let Some(bb) = self.body.basic_blocks.get(bb) { + let src = self.body.basic_blocks.get(location.block).unwrap(); match (src.is_cleanup, bb.is_cleanup, edge_kind) { // Non-cleanup blocks can jump to non-cleanup blocks along non-unwind edges (false, false, EdgeKind::Normal) @@ -183,16 +183,23 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { if (src, dest).has_opaque_types() { return true; } - // Normalize projections and things like that. - let param_env = self.param_env.with_reveal_all_normalized(self.tcx); - let src = self.tcx.normalize_erasing_regions(param_env, src); - let dest = self.tcx.normalize_erasing_regions(param_env, dest); + // Normalize projections and things like that. // Type-changing assignments can happen when subtyping is used. While // all normal lifetimes are erased, higher-ranked types with their // late-bound lifetimes are still around and can lead to type // differences. So we compare ignoring lifetimes. - equal_up_to_regions(self.tcx, param_env, src, dest) + + // First, try with reveal_all. This might not work in some cases, as the predicates + // can be cleared in reveal_all mode. We try the reveal first anyways as it is used + // by some other passes like inlining as well. + let param_env = self.param_env.with_reveal_all_normalized(self.tcx); + if equal_up_to_regions(self.tcx, param_env, src, dest) { + return true; + } + + // If this fails, we can try it without the reveal. + equal_up_to_regions(self.tcx, self.param_env, src, dest) } } @@ -223,7 +230,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed. - if self.tcx.sess.opts.unstable_opts.validate_mir && self.mir_phase < MirPhase::DropsLowered + if self.tcx.sess.opts.unstable_opts.validate_mir + && self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { // `Operand::Copy` is only supposed to be used with `Copy` types. if let Operand::Copy(place) = operand { @@ -254,7 +262,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.fail(location, format!("bad index ({:?} != usize)", index_ty)) } } - ProjectionElem::Deref if self.mir_phase >= MirPhase::GeneratorsLowered => { + ProjectionElem::Deref + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) => + { let base_ty = Place::ty_from(local, proj_base, &self.body.local_decls, self.tcx).ty; if base_ty.is_box() { @@ -362,7 +372,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { // Set off any `bug!`s in the type computation code let _ = place.ty(&self.body.local_decls, self.tcx); - if self.mir_phase >= MirPhase::Derefered + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) && place.projection.len() > 1 && cntxt != PlaceContext::NonUse(VarDebugInfo) && place.projection[1..].contains(&ProjectionElem::Deref) @@ -386,8 +396,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { Rvalue::Aggregate(agg_kind, _) => { let disallowed = match **agg_kind { AggregateKind::Array(..) => false, - AggregateKind::Generator(..) => self.mir_phase >= MirPhase::GeneratorsLowered, - _ => self.mir_phase >= MirPhase::Deaggregated, + _ => self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup), }; if disallowed { self.fail( @@ -397,10 +406,10 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } Rvalue::Ref(_, BorrowKind::Shallow, _) => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, - "`Assign` statement with a `Shallow` borrow should have been removed after drop lowering phase", + "`Assign` statement with a `Shallow` borrow should have been removed in runtime MIR", ); } } @@ -560,6 +569,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } } + CastKind::DynStar => { + // FIXME(dyn-star): make sure nothing needs to be done here. + } // Nothing to check here CastKind::PointerFromExposedAddress | CastKind::PointerExposeAddress @@ -614,7 +626,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::AscribeUserType(..) => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`AscribeUserType` should have been removed after drop lowering phase", @@ -622,18 +634,25 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::FakeRead(..) => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FakeRead` should have been removed after drop lowering phase", ); } } - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { - ref src, - ref dst, - ref count, - }) => { + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { + let ty = op.ty(&self.body.local_decls, self.tcx); + if !ty.is_bool() { + self.fail( + location, + format!("`assume` argument must be `bool`, but got: `{}`", ty), + ); + } + } + StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping( + CopyNonOverlapping { src, dst, count }, + )) => { let src_ty = src.ty(&self.body.local_decls, self.tcx); let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) { src_deref.ty @@ -666,7 +685,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::SetDiscriminant { place, .. } => { - if self.mir_phase < MirPhase::Deaggregated { + if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); } let pty = place.ty(&self.body.local_decls, self.tcx).ty.kind(); @@ -681,7 +700,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::Deinit(..) => { - if self.mir_phase < MirPhase::Deaggregated { + if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`Deinit`is not allowed until deaggregation"); } } @@ -761,7 +780,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } TerminatorKind::DropAndReplace { target, unwind, .. } => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`DropAndReplace` should have been removed during drop elaboration", @@ -832,7 +851,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { if self.body.generator.is_none() { self.fail(location, "`Yield` cannot appear outside generator bodies"); } - if self.mir_phase >= MirPhase::GeneratorsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`Yield` should have been replaced by generator lowering"); } self.check_edge(location, *resume, EdgeKind::Normal); @@ -841,7 +860,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } TerminatorKind::FalseEdge { real_target, imaginary_target } => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FalseEdge` should have been removed after drop elaboration", @@ -851,7 +870,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.check_edge(location, *imaginary_target, EdgeKind::Normal); } TerminatorKind::FalseUnwind { real_target, unwind } => { - if self.mir_phase >= MirPhase::DropsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FalseUnwind` should have been removed after drop elaboration", @@ -874,7 +893,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { if self.body.generator.is_none() { self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies"); } - if self.mir_phase >= MirPhase::GeneratorsLowered { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`GeneratorDrop` should have been replaced by generator lowering", @@ -883,13 +902,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } TerminatorKind::Resume | TerminatorKind::Abort => { let bb = location.block; - if !self.body.basic_blocks()[bb].is_cleanup { + if !self.body.basic_blocks[bb].is_cleanup { self.fail(location, "Cannot `Resume` or `Abort` from non-cleanup basic block") } } TerminatorKind::Return => { let bb = location.block; - if self.body.basic_blocks()[bb].is_cleanup { + if self.body.basic_blocks[bb].is_cleanup { self.fail(location, "Cannot `Return` from cleanup basic block") } } |