summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_const_eval/src
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_const_eval/src')
-rw-r--r--compiler/rustc_const_eval/src/const_eval/error.rs8
-rw-r--r--compiler/rustc_const_eval/src/const_eval/eval_queries.rs40
-rw-r--r--compiler/rustc_const_eval/src/const_eval/machine.rs117
-rw-r--r--compiler/rustc_const_eval/src/const_eval/mod.rs21
-rw-r--r--compiler/rustc_const_eval/src/const_eval/valtrees.rs35
-rw-r--r--compiler/rustc_const_eval/src/errors.rs125
-rw-r--r--compiler/rustc_const_eval/src/interpret/cast.rs39
-rw-r--r--compiler/rustc_const_eval/src/interpret/eval_context.rs13
-rw-r--r--compiler/rustc_const_eval/src/interpret/intern.rs16
-rw-r--r--compiler/rustc_const_eval/src/interpret/intrinsics.rs71
-rw-r--r--compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs2
-rw-r--r--compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs2
-rw-r--r--compiler/rustc_const_eval/src/interpret/machine.rs66
-rw-r--r--compiler/rustc_const_eval/src/interpret/memory.rs111
-rw-r--r--compiler/rustc_const_eval/src/interpret/operand.rs234
-rw-r--r--compiler/rustc_const_eval/src/interpret/operator.rs18
-rw-r--r--compiler/rustc_const_eval/src/interpret/place.rs35
-rw-r--r--compiler/rustc_const_eval/src/interpret/projection.rs9
-rw-r--r--compiler/rustc_const_eval/src/interpret/step.rs14
-rw-r--r--compiler/rustc_const_eval/src/interpret/terminator.rs34
-rw-r--r--compiler/rustc_const_eval/src/interpret/traits.rs2
-rw-r--r--compiler/rustc_const_eval/src/interpret/validity.rs259
-rw-r--r--compiler/rustc_const_eval/src/lib.rs2
-rw-r--r--compiler/rustc_const_eval/src/might_permit_raw_init.rs6
-rw-r--r--compiler/rustc_const_eval/src/transform/check_consts/check.rs8
-rw-r--r--compiler/rustc_const_eval/src/transform/check_consts/ops.rs161
-rw-r--r--compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs66
-rw-r--r--compiler/rustc_const_eval/src/transform/promote_consts.rs30
-rw-r--r--compiler/rustc_const_eval/src/transform/validate.rs93
29 files changed, 861 insertions, 776 deletions
diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs
index 322bfd5ce..09d53331b 100644
--- a/compiler/rustc_const_eval/src/const_eval/error.rs
+++ b/compiler/rustc_const_eval/src/const_eval/error.rs
@@ -9,13 +9,13 @@ use rustc_span::{Span, Symbol};
use super::InterpCx;
use crate::interpret::{
- struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType, UnsupportedOpInfo,
+ struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType,
+ UnsupportedOpInfo,
};
/// The CTFE machine has some custom error kinds.
#[derive(Clone, Debug)]
pub enum ConstEvalErrKind {
- NeedsRfc(String),
ConstAccessesStatic,
ModifiedGlobal,
AssertFailure(AssertKind<ConstInt>),
@@ -42,9 +42,6 @@ impl fmt::Display for ConstEvalErrKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use self::ConstEvalErrKind::*;
match *self {
- NeedsRfc(ref msg) => {
- write!(f, "\"{}\" needs an rfc before being allowed inside constants", msg)
- }
ConstAccessesStatic => write!(f, "constant accesses static"),
ModifiedGlobal => {
write!(f, "modifying a static's initial value from another static's initializer")
@@ -158,6 +155,7 @@ impl<'tcx> ConstEvalErr<'tcx> {
InterpError::Unsupported(
UnsupportedOpInfo::ReadPointerAsBytes
| UnsupportedOpInfo::PartialPointerOverwrite(_)
+ | UnsupportedOpInfo::PartialPointerCopy(_),
) => {
err.help("this code performed an operation that depends on the underlying bytes representing a pointer");
err.help("the absolute address of a pointer is not known at compile-time, so such operations are not supported");
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 975fb4b22..a2f14e753 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -2,8 +2,8 @@ use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr};
use crate::interpret::eval_nullary_intrinsic;
use crate::interpret::{
intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId,
- Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking,
- ScalarMaybeUninit, StackPopCleanup, InterpError,
+ Immediate, InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy,
+ RefTracking, StackPopCleanup,
};
use rustc_hir::def::DefKind;
@@ -74,14 +74,16 @@ fn eval_body_using_ecx<'mir, 'tcx>(
None => InternKind::Constant,
}
};
+ ecx.machine.check_alignment = false; // interning doesn't need to respect alignment
intern_const_alloc_recursive(ecx, intern_kind, &ret)?;
+ // we leave alignment checks off, since this `ecx` will not be used for further evaluation anyway
debug!("eval_body_using_ecx done: {:?}", *ret);
Ok(ret)
}
/// The `InterpCx` is only meant to be used to do field and index projections into constants for
-/// `simd_shuffle` and const patterns in match arms.
+/// `simd_shuffle` and const patterns in match arms. It never performs alignment checks.
///
/// The function containing the `match` that is currently being analyzed may have generic bounds
/// that inform us about the generic bounds of the constant. E.g., using an associated constant
@@ -98,7 +100,11 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>(
tcx,
root_span,
param_env,
- CompileTimeInterpreter::new(tcx.const_eval_limit(), can_access_statics),
+ CompileTimeInterpreter::new(
+ tcx.const_eval_limit(),
+ can_access_statics,
+ /*check_alignment:*/ false,
+ ),
)
}
@@ -166,10 +172,7 @@ pub(super) fn op_to_const<'tcx>(
// see comment on `let try_as_immediate` above
Err(imm) => match *imm {
_ if imm.layout.is_zst() => ConstValue::ZeroSized,
- Immediate::Scalar(x) => match x {
- ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s),
- ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place()),
- },
+ Immediate::Scalar(x) => ConstValue::Scalar(x),
Immediate::ScalarPair(a, b) => {
debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
// We know `offset` is relative to the allocation, so we can use `into_parts`.
@@ -194,7 +197,7 @@ pub(super) fn op_to_const<'tcx>(
}
}
-#[instrument(skip(tcx), level = "debug")]
+#[instrument(skip(tcx), level = "debug", ret)]
pub(crate) fn turn_into_const_value<'tcx>(
tcx: TyCtxt<'tcx>,
constant: ConstAlloc<'tcx>,
@@ -203,7 +206,13 @@ pub(crate) fn turn_into_const_value<'tcx>(
let cid = key.value;
let def_id = cid.instance.def.def_id();
let is_static = tcx.is_static(def_id);
- let ecx = mk_eval_cx(tcx, tcx.def_span(key.value.instance.def_id()), key.param_env, is_static);
+ // This is just accessing an already computed constant, so no need to check alginment here.
+ let ecx = mk_eval_cx(
+ tcx,
+ tcx.def_span(key.value.instance.def_id()),
+ key.param_env,
+ /*can_access_statics:*/ is_static,
+ );
let mplace = ecx.raw_const_to_mplace(constant).expect(
"can only fail if layout computation failed, \
@@ -215,10 +224,7 @@ pub(crate) fn turn_into_const_value<'tcx>(
);
// Turn this into a proper constant.
- let const_val = op_to_const(&ecx, &mplace.into());
- debug!(?const_val);
-
- const_val
+ op_to_const(&ecx, &mplace.into())
}
#[instrument(skip(tcx), level = "debug")]
@@ -300,7 +306,11 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
key.param_env,
// Statics (and promoteds inside statics) may access other statics, because unlike consts
// they do not have to behave "as if" they were evaluated at runtime.
- CompileTimeInterpreter::new(tcx.const_eval_limit(), /*can_access_statics:*/ is_static),
+ CompileTimeInterpreter::new(
+ tcx.const_eval_limit(),
+ /*can_access_statics:*/ is_static,
+ /*check_alignment:*/ tcx.sess.opts.unstable_opts.extra_const_ub_checks,
+ ),
);
let res = ecx.load_mir(cid.instance.def, cid.promoted);
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index fc2e6652a..e5acacd91 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -35,21 +35,7 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
// All `#[rustc_do_not_const_check]` functions should be hooked here.
let def_id = instance.def_id();
- if Some(def_id) == self.tcx.lang_items().const_eval_select() {
- // redirect to const_eval_select_ct
- if let Some(const_eval_select) = self.tcx.lang_items().const_eval_select_ct() {
- return Ok(Some(
- ty::Instance::resolve(
- *self.tcx,
- ty::ParamEnv::reveal_all(),
- const_eval_select,
- instance.substs,
- )
- .unwrap()
- .unwrap(),
- ));
- }
- } else if Some(def_id) == self.tcx.lang_items().panic_display()
+ if Some(def_id) == self.tcx.lang_items().panic_display()
|| Some(def_id) == self.tcx.lang_items().begin_panic_fn()
{
// &str or &&str
@@ -89,10 +75,10 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// exhaustion error.
///
/// Setting this to `0` disables the limit and allows the interpreter to run forever.
- pub steps_remaining: usize,
+ pub(super) steps_remaining: usize,
/// The virtual call stack.
- pub(crate) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>,
+ pub(super) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>,
/// We need to make sure consts never point to anything mutable, even recursively. That is
/// relied on for pattern matching on consts with references.
@@ -101,14 +87,22 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// * Pointers to allocations inside of statics can never leak outside, to a non-static global.
/// This boolean here controls the second part.
pub(super) can_access_statics: bool,
+
+ /// Whether to check alignment during evaluation.
+ pub(super) check_alignment: bool,
}
impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
- pub(crate) fn new(const_eval_limit: Limit, can_access_statics: bool) -> Self {
+ pub(crate) fn new(
+ const_eval_limit: Limit,
+ can_access_statics: bool,
+ check_alignment: bool,
+ ) -> Self {
CompileTimeInterpreter {
steps_remaining: const_eval_limit.0,
stack: Vec::new(),
can_access_statics,
+ check_alignment,
}
}
}
@@ -197,34 +191,35 @@ impl interpret::MayLeak for ! {
}
impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
- fn guaranteed_eq(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> {
+ /// See documentation on the `ptr_guaranteed_cmp` intrinsic.
+ fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> {
Ok(match (a, b) {
// Comparisons between integers are always known.
- (Scalar::Int { .. }, Scalar::Int { .. }) => a == b,
- // Equality with integers can never be known for sure.
- (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => false,
- // FIXME: return `true` for when both sides are the same pointer, *except* that
- // some things (like functions and vtables) do not have stable addresses
- // so we need to be careful around them (see e.g. #73722).
- (Scalar::Ptr(..), Scalar::Ptr(..)) => false,
- })
- }
-
- fn guaranteed_ne(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, bool> {
- Ok(match (a, b) {
- // Comparisons between integers are always known.
- (Scalar::Int(_), Scalar::Int(_)) => a != b,
+ (Scalar::Int { .. }, Scalar::Int { .. }) => {
+ if a == b {
+ 1
+ } else {
+ 0
+ }
+ }
// Comparisons of abstract pointers with null pointers are known if the pointer
// is in bounds, because if they are in bounds, the pointer can't be null.
// Inequality with integers other than null can never be known for sure.
(Scalar::Int(int), ptr @ Scalar::Ptr(..))
- | (ptr @ Scalar::Ptr(..), Scalar::Int(int)) => {
- int.is_null() && !self.scalar_may_be_null(ptr)?
+ | (ptr @ Scalar::Ptr(..), Scalar::Int(int))
+ if int.is_null() && !self.scalar_may_be_null(ptr)? =>
+ {
+ 0
}
- // FIXME: return `true` for at least some comparisons where we can reliably
+ // Equality with integers can never be known for sure.
+ (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => 2,
+ // FIXME: return a `1` for when both sides are the same pointer, *except* that
+ // some things (like functions and vtables) do not have stable addresses
+ // so we need to be careful around them (see e.g. #73722).
+ // FIXME: return `0` for at least some comparisons where we can reliably
// determine the result of runtime inequality tests at compile-time.
// Examples include comparison of addresses in different static items.
- (Scalar::Ptr(..), Scalar::Ptr(..)) => false,
+ (Scalar::Ptr(..), Scalar::Ptr(..)) => 2,
})
}
}
@@ -236,6 +231,16 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
const PANIC_ON_ALLOC_FAIL: bool = false; // will be raised as a proper error
+ #[inline(always)]
+ fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+ ecx.machine.check_alignment
+ }
+
+ #[inline(always)]
+ fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+ ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks
+ }
+
fn load_mir(
ecx: &InterpCx<'mir, 'tcx, Self>,
instance: ty::InstanceDef<'tcx>,
@@ -251,9 +256,10 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
);
throw_inval!(AlreadyReported(guar));
} else {
+ // `find_mir_or_eval_fn` checks that this is a const fn before even calling us,
+ // so this should be unreachable.
let path = ecx.tcx.def_path_str(def.did);
- Err(ConstEvalErrKind::NeedsRfc(format!("calling extern function `{}`", path))
- .into())
+ bug!("trying to call extern function `{path}` at compile-time");
}
}
_ => Ok(ecx.tcx.instance_mir(instance)),
@@ -321,22 +327,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
// CTFE-specific intrinsics.
let Some(ret) = target else {
- return Err(ConstEvalErrKind::NeedsRfc(format!(
- "calling intrinsic `{}`",
- intrinsic_name
- ))
- .into());
+ throw_unsup_format!("intrinsic `{intrinsic_name}` is not supported at compile-time");
};
match intrinsic_name {
- sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
- let a = ecx.read_immediate(&args[0])?.to_scalar()?;
- let b = ecx.read_immediate(&args[1])?.to_scalar()?;
- let cmp = if intrinsic_name == sym::ptr_guaranteed_eq {
- ecx.guaranteed_eq(a, b)?
- } else {
- ecx.guaranteed_ne(a, b)?
- };
- ecx.write_scalar(Scalar::from_bool(cmp), dest)?;
+ sym::ptr_guaranteed_cmp => {
+ let a = ecx.read_scalar(&args[0])?;
+ let b = ecx.read_scalar(&args[1])?;
+ let cmp = ecx.guaranteed_cmp(a, b)?;
+ ecx.write_scalar(Scalar::from_u8(cmp), dest)?;
}
sym::const_allocate => {
let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?;
@@ -382,11 +380,9 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
}
}
_ => {
- return Err(ConstEvalErrKind::NeedsRfc(format!(
- "calling intrinsic `{}`",
- intrinsic_name
- ))
- .into());
+ throw_unsup_format!(
+ "intrinsic `{intrinsic_name}` is not supported at compile-time"
+ );
}
}
@@ -429,7 +425,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
_left: &ImmTy<'tcx>,
_right: &ImmTy<'tcx>,
) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> {
- Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into())
+ throw_unsup_format!("pointer arithmetic or comparison is not supported at compile-time");
}
fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
@@ -451,7 +447,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
_ecx: &mut InterpCx<'mir, 'tcx, Self>,
_ptr: Pointer<AllocId>,
) -> InterpResult<'tcx> {
- Err(ConstEvalErrKind::NeedsRfc("exposing pointers".to_string()).into())
+ // This is only reachable with -Zunleash-the-miri-inside-of-you.
+ throw_unsup_format!("exposing pointers is not possible at compile-time")
}
#[inline(always)]
diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs
index 948c33494..d9c4ae4d5 100644
--- a/compiler/rustc_const_eval/src/const_eval/mod.rs
+++ b/compiler/rustc_const_eval/src/const_eval/mod.rs
@@ -1,16 +1,16 @@
// Not in interpret to make sure we do not use private implementation details
+use crate::errors::MaxNumNodesInConstErr;
+use crate::interpret::{
+ intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, InterpResult, MemPlaceMeta,
+ Scalar,
+};
use rustc_hir::Mutability;
use rustc_middle::mir;
use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId};
use rustc_middle::ty::{self, TyCtxt};
use rustc_span::{source_map::DUMMY_SP, symbol::Symbol};
-use crate::interpret::{
- intern_const_alloc_recursive, ConstValue, InternKind, InterpCx, InterpResult, MemPlaceMeta,
- Scalar,
-};
-
mod error;
mod eval_queries;
mod fn_queries;
@@ -72,12 +72,17 @@ pub(crate) fn eval_to_valtree<'tcx>(
Ok(valtree) => Ok(Some(valtree)),
Err(err) => {
let did = cid.instance.def_id();
- let s = cid.display(tcx);
+ let global_const_id = cid.display(tcx);
match err {
ValTreeCreationError::NodesOverflow => {
- let msg = format!("maximum number of nodes exceeded in constant {}", &s);
+ let msg = format!(
+ "maximum number of nodes exceeded in constant {}",
+ &global_const_id
+ );
let mut diag = match tcx.hir().span_if_local(did) {
- Some(span) => tcx.sess.struct_span_err(span, &msg),
+ Some(span) => {
+ tcx.sess.create_err(MaxNumNodesInConstErr { span, global_const_id })
+ }
None => tcx.sess.struct_err(&msg),
};
diag.emit();
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index 8fff4571d..a964fe846 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -3,7 +3,7 @@ use super::machine::CompileTimeEvalContext;
use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES};
use crate::interpret::{
intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta,
- MemoryKind, PlaceTy, Scalar, ScalarMaybeUninit,
+ MemoryKind, PlaceTy, Scalar,
};
use crate::interpret::{MPlaceTy, Value};
use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
@@ -90,14 +90,14 @@ pub(crate) fn const_to_valtree_inner<'tcx>(
let Ok(val) = ecx.read_immediate(&place.into()) else {
return Err(ValTreeCreationError::Other);
};
- let val = val.to_scalar().unwrap();
+ let val = val.to_scalar();
*num_nodes += 1;
Ok(ty::ValTree::Leaf(val.assert_int()))
}
// Raw pointers are not allowed in type level constants, as we cannot properly test them for
- // equality at compile-time (see `ptr_guaranteed_eq`/`_ne`).
+ // equality at compile-time (see `ptr_guaranteed_cmp`).
// Technically we could allow function pointers (represented as `ty::Instance`), but this is not guaranteed to
// agree with runtime equality tests.
ty::FnPtr(_) | ty::RawPtr(_) => Err(ValTreeCreationError::NonSupportedType),
@@ -204,7 +204,7 @@ fn get_info_on_unsized_field<'tcx>(
(unsized_inner_ty, num_elems)
}
-#[instrument(skip(ecx), level = "debug")]
+#[instrument(skip(ecx), level = "debug", ret)]
fn create_pointee_place<'tcx>(
ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>,
ty: Ty<'tcx>,
@@ -237,14 +237,11 @@ fn create_pointee_place<'tcx>(
let ptr = ecx.allocate_ptr(size, align, MemoryKind::Stack).unwrap();
debug!(?ptr);
- let place = MPlaceTy::from_aligned_ptr_with_meta(
+ MPlaceTy::from_aligned_ptr_with_meta(
ptr.into(),
layout,
MemPlaceMeta::Meta(Scalar::from_machine_usize(num_elems as u64, &tcx)),
- );
- debug!(?place);
-
- place
+ )
} else {
create_mplace_from_layout(ecx, ty)
}
@@ -253,7 +250,7 @@ fn create_pointee_place<'tcx>(
/// Converts a `ValTree` to a `ConstValue`, which is needed after mir
/// construction has finished.
// FIXME Merge `valtree_to_const_value` and `valtree_into_mplace` into one function
-#[instrument(skip(tcx), level = "debug")]
+#[instrument(skip(tcx), level = "debug", ret)]
pub fn valtree_to_const_value<'tcx>(
tcx: TyCtxt<'tcx>,
param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
@@ -294,7 +291,7 @@ pub fn valtree_to_const_value<'tcx>(
dump_place(&ecx, place.into());
intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &place).unwrap();
- let const_val = match ty.kind() {
+ match ty.kind() {
ty::Ref(_, _, _) => {
let ref_place = place.to_ref(&tcx);
let imm =
@@ -303,10 +300,7 @@ pub fn valtree_to_const_value<'tcx>(
op_to_const(&ecx, &imm.into())
}
_ => op_to_const(&ecx, &place.into()),
- };
- debug!(?const_val);
-
- const_val
+ }
}
ty::Never
| ty::Error(_)
@@ -349,11 +343,7 @@ fn valtree_into_mplace<'tcx>(
ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
let scalar_int = valtree.unwrap_leaf();
debug!("writing trivial valtree {:?} to place {:?}", scalar_int, place);
- ecx.write_immediate(
- Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar_int.into())),
- &place.into(),
- )
- .unwrap();
+ ecx.write_immediate(Immediate::Scalar(scalar_int.into()), &place.into()).unwrap();
}
ty::Ref(_, inner_ty, _) => {
let mut pointee_place = create_pointee_place(ecx, *inner_ty, valtree);
@@ -366,11 +356,10 @@ fn valtree_into_mplace<'tcx>(
let imm = match inner_ty.kind() {
ty::Slice(_) | ty::Str => {
let len = valtree.unwrap_branch().len();
- let len_scalar =
- ScalarMaybeUninit::Scalar(Scalar::from_machine_usize(len as u64, &tcx));
+ let len_scalar = Scalar::from_machine_usize(len as u64, &tcx);
Immediate::ScalarPair(
- ScalarMaybeUninit::from_maybe_pointer((*pointee_place).ptr, &tcx),
+ Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx),
len_scalar,
)
}
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index a463fe7b9..c3547cb3a 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -3,7 +3,7 @@ use rustc_macros::SessionDiagnostic;
use rustc_span::Span;
#[derive(SessionDiagnostic)]
-#[error(const_eval::unstable_in_stable)]
+#[diag(const_eval::unstable_in_stable)]
pub(crate) struct UnstableInStable {
pub gate: String,
#[primary_span]
@@ -22,14 +22,14 @@ pub(crate) struct UnstableInStable {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::thread_local_access, code = "E0625")]
+#[diag(const_eval::thread_local_access, code = "E0625")]
pub(crate) struct NonConstOpErr {
#[primary_span]
pub span: Span,
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::static_access, code = "E0013")]
+#[diag(const_eval::static_access, code = "E0013")]
#[help]
pub(crate) struct StaticAccessErr {
#[primary_span]
@@ -41,7 +41,7 @@ pub(crate) struct StaticAccessErr {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::raw_ptr_to_int)]
+#[diag(const_eval::raw_ptr_to_int)]
#[note]
#[note(const_eval::note2)]
pub(crate) struct RawPtrToIntErr {
@@ -50,7 +50,7 @@ pub(crate) struct RawPtrToIntErr {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::raw_ptr_comparison)]
+#[diag(const_eval::raw_ptr_comparison)]
#[note]
pub(crate) struct RawPtrComparisonErr {
#[primary_span]
@@ -58,14 +58,14 @@ pub(crate) struct RawPtrComparisonErr {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::panic_non_str)]
+#[diag(const_eval::panic_non_str)]
pub(crate) struct PanicNonStrErr {
#[primary_span]
pub span: Span,
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::mut_deref, code = "E0658")]
+#[diag(const_eval::mut_deref, code = "E0658")]
pub(crate) struct MutDerefErr {
#[primary_span]
pub span: Span,
@@ -73,7 +73,7 @@ pub(crate) struct MutDerefErr {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::transient_mut_borrow, code = "E0658")]
+#[diag(const_eval::transient_mut_borrow, code = "E0658")]
pub(crate) struct TransientMutBorrowErr {
#[primary_span]
pub span: Span,
@@ -81,9 +81,116 @@ pub(crate) struct TransientMutBorrowErr {
}
#[derive(SessionDiagnostic)]
-#[error(const_eval::transient_mut_borrow_raw, code = "E0658")]
+#[diag(const_eval::transient_mut_borrow_raw, code = "E0658")]
pub(crate) struct TransientMutBorrowErrRaw {
#[primary_span]
pub span: Span,
pub kind: ConstContext,
}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::max_num_nodes_in_const)]
+pub(crate) struct MaxNumNodesInConstErr {
+ #[primary_span]
+ pub span: Span,
+ pub global_const_id: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_fn_pointer_call)]
+pub(crate) struct UnallowedFnPointerCall {
+ #[primary_span]
+ pub span: Span,
+ pub kind: ConstContext,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unstable_const_fn)]
+pub(crate) struct UnstableConstFn {
+ #[primary_span]
+ pub span: Span,
+ pub def_path: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_mutable_refs, code = "E0764")]
+pub(crate) struct UnallowedMutableRefs {
+ #[primary_span]
+ pub span: Span,
+ pub kind: ConstContext,
+ #[note(const_eval::teach_note)]
+ pub teach: Option<()>,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_mutable_refs_raw, code = "E0764")]
+pub(crate) struct UnallowedMutableRefsRaw {
+ #[primary_span]
+ pub span: Span,
+ pub kind: ConstContext,
+ #[note(const_eval::teach_note)]
+ pub teach: Option<()>,
+}
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::non_const_fmt_macro_call, code = "E0015")]
+pub(crate) struct NonConstFmtMacroCall {
+ #[primary_span]
+ pub span: Span,
+ pub kind: ConstContext,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::non_const_fn_call, code = "E0015")]
+pub(crate) struct NonConstFnCall {
+ #[primary_span]
+ pub span: Span,
+ pub def_path_str: String,
+ pub kind: ConstContext,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_op_in_const_context)]
+pub(crate) struct UnallowedOpInConstContext {
+ #[primary_span]
+ pub span: Span,
+ pub msg: String,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_heap_allocations, code = "E0010")]
+pub(crate) struct UnallowedHeapAllocations {
+ #[primary_span]
+ #[label]
+ pub span: Span,
+ pub kind: ConstContext,
+ #[note(const_eval::teach_note)]
+ pub teach: Option<()>,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::unallowed_inline_asm, code = "E0015")]
+pub(crate) struct UnallowedInlineAsm {
+ #[primary_span]
+ pub span: Span,
+ pub kind: ConstContext,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::interior_mutable_data_refer, code = "E0492")]
+pub(crate) struct InteriorMutableDataRefer {
+ #[primary_span]
+ #[label]
+ pub span: Span,
+ #[help]
+ pub opt_help: Option<()>,
+ pub kind: ConstContext,
+ #[note(const_eval::teach_note)]
+ pub teach: Option<()>,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(const_eval::interior_mutability_borrow)]
+pub(crate) struct InteriorMutabilityBorrow {
+ #[primary_span]
+ pub span: Span,
+}
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index c97c31eb9..cbe985480 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -108,6 +108,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
_ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty),
}
}
+
+ DynStar => {
+ if let ty::Dynamic(data, _, ty::DynStar) = cast_ty.kind() {
+ // Initial cast from sized to dyn trait
+ let vtable = self.get_vtable_ptr(src.layout.ty, data.principal())?;
+ let vtable = Scalar::from_maybe_pointer(vtable, self);
+ let data = self.read_immediate(src)?.to_scalar();
+ let _assert_pointer_sized = data.to_pointer(self)?;
+ let val = Immediate::ScalarPair(data, vtable);
+ self.write_immediate(val, dest)?;
+ } else {
+ bug!()
+ }
+ }
}
Ok(())
}
@@ -123,10 +137,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
match src.layout.ty.kind() {
// Floating point
Float(FloatTy::F32) => {
- return Ok(self.cast_from_float(src.to_scalar()?.to_f32()?, cast_ty).into());
+ return Ok(self.cast_from_float(src.to_scalar().to_f32()?, cast_ty).into());
}
Float(FloatTy::F64) => {
- return Ok(self.cast_from_float(src.to_scalar()?.to_f64()?, cast_ty).into());
+ return Ok(self.cast_from_float(src.to_scalar().to_f64()?, cast_ty).into());
}
// The rest is integer/pointer-"like", including fn ptr casts
_ => assert!(
@@ -153,7 +167,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert_eq!(dest_layout.size, self.pointer_size());
assert!(src.layout.ty.is_unsafe_ptr());
return match **src {
- Immediate::ScalarPair(data, _) => Ok(data.check_init()?.into()),
+ Immediate::ScalarPair(data, _) => Ok(data.into()),
Immediate::Scalar(..) => span_bug!(
self.cur_span(),
"{:?} input to a fat-to-thin cast ({:?} -> {:?})",
@@ -167,7 +181,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
// # The remaining source values are scalar and "int-like".
- let scalar = src.to_scalar()?;
+ let scalar = src.to_scalar();
Ok(self.cast_from_int_like(scalar, src.layout, cast_ty)?.into())
}
@@ -179,7 +193,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert_matches!(src.layout.ty.kind(), ty::RawPtr(_) | ty::FnPtr(_));
assert!(cast_ty.is_integral());
- let scalar = src.to_scalar()?;
+ let scalar = src.to_scalar();
let ptr = scalar.to_pointer(self)?;
match ptr.into_pointer_or_addr() {
Ok(ptr) => M::expose_ptr(self, ptr)?,
@@ -197,7 +211,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert_matches!(cast_ty.kind(), ty::RawPtr(_));
// First cast to usize.
- let scalar = src.to_scalar()?;
+ let scalar = src.to_scalar();
let addr = self.cast_from_int_like(scalar, src.layout, self.tcx.types.usize)?;
let addr = addr.to_machine_usize(self)?;
@@ -291,14 +305,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
match (&src_pointee_ty.kind(), &dest_pointee_ty.kind()) {
(&ty::Array(_, length), &ty::Slice(_)) => {
- let ptr = self.read_immediate(src)?.to_scalar()?;
+ let ptr = self.read_scalar(src)?;
// u64 cast is from usize to u64, which is always good
let val =
Immediate::new_slice(ptr, length.eval_usize(*self.tcx, self.param_env), self);
self.write_immediate(val, dest)
}
(&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => {
- let (old_data, old_vptr) = self.read_immediate(src)?.to_scalar_pair()?;
+ let val = self.read_immediate(src)?;
+ if data_a.principal() == data_b.principal() {
+ // A NOP cast that doesn't actually change anything, should be allowed even with mismatching vtables.
+ return self.write_immediate(*val, dest);
+ }
+ let (old_data, old_vptr) = val.to_scalar_pair();
let old_vptr = old_vptr.to_pointer(self)?;
let (ty, old_trait) = self.get_ptr_vtable(old_vptr)?;
if old_trait != data_a.principal() {
@@ -307,10 +326,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let new_vptr = self.get_vtable_ptr(ty, data_b.principal())?;
self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest)
}
- (_, &ty::Dynamic(ref data, _)) => {
+ (_, &ty::Dynamic(ref data, _, ty::Dyn)) => {
// Initial cast from sized to dyn trait
let vtable = self.get_vtable_ptr(src_pointee_ty, data.principal())?;
- let ptr = self.read_immediate(src)?.to_scalar()?;
+ let ptr = self.read_scalar(src)?;
let val = Immediate::new_dyn_trait(ptr, vtable, &*self.tcx);
self.write_immediate(val, dest)
}
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index 150d6589b..d37eaeed0 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -21,7 +21,7 @@ use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayou
use super::{
AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace,
MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance,
- Scalar, ScalarMaybeUninit, StackPopJump,
+ Scalar, StackPopJump,
};
use crate::transform::validate::equal_up_to_regions;
@@ -187,9 +187,6 @@ pub enum LocalValue<Prov: Provenance = AllocId> {
impl<'tcx, Prov: Provenance + 'static> LocalState<'tcx, Prov> {
/// Read the local's value or error if the local is not yet live or not live anymore.
- ///
- /// Note: This may only be invoked from the `Machine::access_local` hook and not from
- /// anywhere else. You may be invalidating machine invariants if you do!
#[inline]
pub fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
match &self.value {
@@ -782,7 +779,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert_eq!(
unwinding,
match self.frame().loc {
- Ok(loc) => self.body().basic_blocks()[loc.block].is_cleanup,
+ Ok(loc) => self.body().basic_blocks[loc.block].is_cleanup,
Err(_) => true,
}
);
@@ -991,16 +988,16 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> std::fmt::Debug
}
LocalValue::Live(Operand::Immediate(Immediate::Scalar(val))) => {
write!(fmt, " {:?}", val)?;
- if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val {
+ if let Scalar::Ptr(ptr, _size) = val {
allocs.push(ptr.provenance.get_alloc_id());
}
}
LocalValue::Live(Operand::Immediate(Immediate::ScalarPair(val1, val2))) => {
write!(fmt, " ({:?}, {:?})", val1, val2)?;
- if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val1 {
+ if let Scalar::Ptr(ptr, _size) = val1 {
allocs.push(ptr.provenance.get_alloc_id());
}
- if let ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _size)) = val2 {
+ if let Scalar::Ptr(ptr, _size) = val2 {
allocs.push(ptr.provenance.get_alloc_id());
}
}
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 376b8872c..24dbc7695 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -134,7 +134,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
alloc.mutability = Mutability::Not;
};
// link the alloc id to the actual allocation
- leftover_allocations.extend(alloc.relocations().iter().map(|&(_, alloc_id)| alloc_id));
+ leftover_allocations.extend(alloc.provenance().iter().map(|&(_, alloc_id)| alloc_id));
let alloc = tcx.intern_const_alloc(alloc);
tcx.set_alloc_id_memory(alloc_id, alloc);
None
@@ -191,10 +191,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
return Ok(true);
};
- // If there are no relocations in this allocation, it does not contain references
+ // If there is no provenance in this allocation, it does not contain references
// that point to another allocation, and we can avoid the interning walk.
if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? {
- if !alloc.has_relocations() {
+ if !alloc.has_provenance() {
return Ok(false);
}
} else {
@@ -233,8 +233,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
}
fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
- // Handle Reference types, as these are the only relocations supported by const eval.
- // Raw pointers (and boxes) are handled by the `leftover_relocations` logic.
+ // Handle Reference types, as these are the only types with provenance supported by const eval.
+ // Raw pointers (and boxes) are handled by the `leftover_allocations` logic.
let tcx = self.ecx.tcx;
let ty = mplace.layout.ty;
if let ty::Ref(_, referenced_ty, ref_mutability) = *ty.kind() {
@@ -334,7 +334,7 @@ pub enum InternKind {
/// tracks where in the value we are and thus can show much better error messages.
/// Any errors here would anyway be turned into `const_err` lints, whereas validation failures
/// are hard errors.
-#[tracing::instrument(level = "debug", skip(ecx))]
+#[instrument(level = "debug", skip(ecx))]
pub fn intern_const_alloc_recursive<
'mir,
'tcx: 'mir,
@@ -410,7 +410,7 @@ pub fn intern_const_alloc_recursive<
// references and a `leftover_allocations` set (where we only have a todo-list here).
// So we hand-roll the interning logic here again.
match intern_kind {
- // Statics may contain mutable allocations even behind relocations.
+ // Statics may point to mutable allocations.
// Even for immutable statics it would be ok to have mutable allocations behind
// raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`.
InternKind::Static(_) => {}
@@ -441,7 +441,7 @@ pub fn intern_const_alloc_recursive<
}
let alloc = tcx.intern_const_alloc(alloc);
tcx.set_alloc_id_memory(alloc_id, alloc);
- for &(_, alloc_id) in alloc.inner().relocations().iter() {
+ for &(_, alloc_id) in alloc.inner().provenance().iter() {
if leftover_allocations.insert(alloc_id) {
todo.push(alloc_id);
}
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index 08209eb79..8637d6a77 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -8,7 +8,7 @@ use rustc_hir::def_id::DefId;
use rustc_middle::mir::{
self,
interpret::{ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar},
- BinOp,
+ BinOp, NonDivergingIntrinsic,
};
use rustc_middle::ty;
use rustc_middle::ty::layout::LayoutOf as _;
@@ -79,9 +79,9 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
ty::Projection(_)
| ty::Opaque(_, _)
| ty::Param(_)
- | ty::Bound(_, _)
| ty::Placeholder(_)
| ty::Infer(_) => throw_inval!(TooGeneric),
+ ty::Bound(_, _) => bug!("bound ty during ctfe"),
ty::Bool
| ty::Char
| ty::Int(_)
@@ -95,7 +95,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
| ty::Ref(_, _, _)
| ty::FnDef(_, _)
| ty::FnPtr(_)
- | ty::Dynamic(_, _)
+ | ty::Dynamic(_, _, _)
| ty::Closure(_, _)
| ty::Generator(_, _, _)
| ty::GeneratorWitness(_)
@@ -184,7 +184,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
| sym::bitreverse => {
let ty = substs.type_at(0);
let layout_of = self.layout_of(ty)?;
- let val = self.read_scalar(&args[0])?.check_init()?;
+ let val = self.read_scalar(&args[0])?;
let bits = val.to_bits(layout_of.size)?;
let kind = match layout_of.abi {
Abi::Scalar(scalar) => scalar.primitive(),
@@ -256,7 +256,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let (val, overflowed, _ty) = self.overflowing_binary_op(bin_op, &l, &r)?;
if overflowed {
let layout = self.layout_of(substs.type_at(0))?;
- let r_val = r.to_scalar()?.to_bits(layout.size)?;
+ let r_val = r.to_scalar().to_bits(layout.size)?;
if let sym::unchecked_shl | sym::unchecked_shr = intrinsic_name {
throw_ub_format!("overflowing shift by {} in `{}`", r_val, intrinsic_name);
} else {
@@ -269,9 +269,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
// rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
let layout = self.layout_of(substs.type_at(0))?;
- let val = self.read_scalar(&args[0])?.check_init()?;
+ let val = self.read_scalar(&args[0])?;
let val_bits = val.to_bits(layout.size)?;
- let raw_shift = self.read_scalar(&args[1])?.check_init()?;
+ let raw_shift = self.read_scalar(&args[1])?;
let raw_shift_bits = raw_shift.to_bits(layout.size)?;
let width_bits = u128::from(layout.size.bits());
let shift_bits = raw_shift_bits % width_bits;
@@ -320,7 +320,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let (a_offset, b_offset) =
match (self.ptr_try_get_alloc_id(a), self.ptr_try_get_alloc_id(b)) {
(Err(a), Err(b)) => {
- // Neither poiner points to an allocation.
+ // Neither pointer points to an allocation.
// If these are inequal or null, this *will* fail the deref check below.
(a, b)
}
@@ -506,12 +506,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// These just return their argument
self.copy_op(&args[0], dest, /*allow_transmute*/ false)?;
}
- sym::assume => {
- let cond = self.read_scalar(&args[0])?.check_init()?.to_bool()?;
- if !cond {
- throw_ub_format!("`assume` intrinsic called with `false`");
- }
- }
sym::raw_eq => {
let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
self.write_scalar(result, dest)?;
@@ -536,6 +530,32 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Ok(true)
}
+ pub(super) fn emulate_nondiverging_intrinsic(
+ &mut self,
+ intrinsic: &NonDivergingIntrinsic<'tcx>,
+ ) -> InterpResult<'tcx> {
+ match intrinsic {
+ NonDivergingIntrinsic::Assume(op) => {
+ let op = self.eval_operand(op, None)?;
+ let cond = self.read_scalar(&op)?.to_bool()?;
+ if !cond {
+ throw_ub_format!("`assume` called with `false`");
+ }
+ Ok(())
+ }
+ NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+ count,
+ src,
+ dst,
+ }) => {
+ let src = self.eval_operand(src, None)?;
+ let dst = self.eval_operand(dst, None)?;
+ let count = self.eval_operand(count, None)?;
+ self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
+ }
+ }
+ }
+
pub fn exact_div(
&mut self,
a: &ImmTy<'tcx, M::Provenance>,
@@ -570,7 +590,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// term since the sign of the second term can be inferred from this and
// the fact that the operation has overflowed (if either is 0 no
// overflow can occur)
- let first_term: u128 = l.to_scalar()?.to_bits(l.layout.size)?;
+ let first_term: u128 = l.to_scalar().to_bits(l.layout.size)?;
let first_term_positive = first_term & (1 << (num_bits - 1)) == 0;
if first_term_positive {
// Negative overflow not possible since the positive first term
@@ -687,10 +707,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap().ty)?;
assert!(!layout.is_unsized());
- let lhs = self.read_pointer(lhs)?;
- let rhs = self.read_pointer(rhs)?;
- let lhs_bytes = self.read_bytes_ptr(lhs, layout.size)?;
- let rhs_bytes = self.read_bytes_ptr(rhs, layout.size)?;
+ let get_bytes = |this: &InterpCx<'mir, 'tcx, M>,
+ op: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>,
+ size|
+ -> InterpResult<'tcx, &[u8]> {
+ let ptr = this.read_pointer(op)?;
+ let Some(alloc_ref) = self.get_ptr_alloc(ptr, size, Align::ONE)? else {
+ // zero-sized access
+ return Ok(&[]);
+ };
+ if alloc_ref.has_provenance() {
+ throw_ub_format!("`raw_eq` on bytes with provenance");
+ }
+ alloc_ref.get_bytes_strip_provenance()
+ };
+
+ let lhs_bytes = get_bytes(self, lhs, layout.size)?;
+ let rhs_bytes = get_bytes(self, rhs, layout.size)?;
Ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}
}
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
index 5864b9215..91f4f0425 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
@@ -28,7 +28,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let mut source_info = *frame.body.source_info(loc);
// If this is a `Call` terminator, use the `fn_span` instead.
- let block = &frame.body.basic_blocks()[loc.block];
+ let block = &frame.body.basic_blocks[loc.block];
if loc.statement_index == block.statements.len() {
debug!(
"find_closest_untracked_caller_location: got terminator {:?} ({:?})",
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
index f9847742f..7e4c5fcb0 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
@@ -48,7 +48,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> {
| ty::FnPtr(_)
| ty::Never
| ty::Tuple(_)
- | ty::Dynamic(_, _) => self.pretty_print_type(ty),
+ | ty::Dynamic(_, _, _) => self.pretty_print_type(ty),
// Placeholders (all printed as `_` to uniformize them).
ty::Param(_) | ty::Bound(..) | ty::Placeholder(_) | ty::Infer(_) | ty::Error(_) => {
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index 71ccd1799..530e252b7 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -6,6 +6,7 @@ use std::borrow::{Borrow, Cow};
use std::fmt::Debug;
use std::hash::Hash;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_middle::mir;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::def_id::DefId;
@@ -123,18 +124,15 @@ pub trait Machine<'mir, 'tcx>: Sized {
/// Whether memory accesses should be alignment-checked.
fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
- /// Whether, when checking alignment, we should `force_int` and thus support
+ /// Whether, when checking alignment, we should look at the actual address and thus support
/// custom alignment logic based on whatever the integer address happens to be.
///
- /// Requires Provenance::OFFSET_IS_ADDR to be true.
- fn force_int_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
+ /// If this returns true, Provenance::OFFSET_IS_ADDR must be true.
+ fn use_addr_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
/// Whether to enforce the validity invariant
fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
- /// Whether to enforce integers and floats being initialized.
- fn enforce_number_init(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
-
/// Whether function calls should be [ABI](CallAbi)-checked.
fn enforce_abi(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
true
@@ -218,23 +216,12 @@ pub trait Machine<'mir, 'tcx>: Sized {
right: &ImmTy<'tcx, Self::Provenance>,
) -> InterpResult<'tcx, (Scalar<Self::Provenance>, bool, Ty<'tcx>)>;
- /// Called to read the specified `local` from the `frame`.
- /// Since reading a ZST is not actually accessing memory or locals, this is never invoked
- /// for ZST reads.
- #[inline]
- fn access_local<'a>(
- frame: &'a Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>,
- local: mir::Local,
- ) -> InterpResult<'tcx, &'a Operand<Self::Provenance>>
- where
- 'tcx: 'mir,
- {
- frame.locals[local].access()
- }
-
/// Called to write the specified `local` from the `frame`.
/// Since writing a ZST is not actually accessing memory or locals, this is never invoked
/// for ZST reads.
+ ///
+ /// Due to borrow checker trouble, we indicate the `frame` as an index rather than an `&mut
+ /// Frame`.
#[inline]
fn access_local_mut<'a>(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
@@ -329,7 +316,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
/// cache the result. (This relies on `AllocMap::get_or` being able to add the
/// owned allocation to the map even when the map is shared.)
///
- /// This must only fail if `alloc` contains relocations.
+ /// This must only fail if `alloc` contains provenance.
fn adjust_allocation<'b>(
ecx: &InterpCx<'mir, 'tcx, Self>,
id: AllocId,
@@ -337,13 +324,22 @@ pub trait Machine<'mir, 'tcx>: Sized {
kind: Option<MemoryKind<Self::MemoryKind>>,
) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra>>>;
+ fn eval_inline_asm(
+ _ecx: &mut InterpCx<'mir, 'tcx, Self>,
+ _template: &'tcx [InlineAsmTemplatePiece],
+ _operands: &[mir::InlineAsmOperand<'tcx>],
+ _options: InlineAsmOptions,
+ ) -> InterpResult<'tcx> {
+ throw_unsup_format!("inline assembly is not supported")
+ }
+
/// Hook for performing extra checks on a memory read access.
///
/// Takes read-only access to the allocation so we can keep all the memory read
/// operations take `&self`. Use a `RefCell` in `AllocExtra` if you
/// need to mutate.
#[inline(always)]
- fn memory_read(
+ fn before_memory_read(
_tcx: TyCtxt<'tcx>,
_machine: &Self,
_alloc_extra: &Self::AllocExtra,
@@ -355,7 +351,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
/// Hook for performing extra checks on a memory write access.
#[inline(always)]
- fn memory_written(
+ fn before_memory_write(
_tcx: TyCtxt<'tcx>,
_machine: &mut Self,
_alloc_extra: &mut Self::AllocExtra,
@@ -367,7 +363,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
/// Hook for performing extra operations on a memory deallocation.
#[inline(always)]
- fn memory_deallocated(
+ fn before_memory_deallocation(
_tcx: TyCtxt<'tcx>,
_machine: &mut Self,
_alloc_extra: &mut Self::AllocExtra,
@@ -437,29 +433,12 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
type FrameExtra = ();
#[inline(always)]
- fn enforce_alignment(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
- // We do not check for alignment to avoid having to carry an `Align`
- // in `ConstValue::ByRef`.
- false
- }
-
- #[inline(always)]
- fn force_int_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
- // We do not support `force_int`.
+ fn use_addr_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
+ // We do not support `use_addr`.
false
}
#[inline(always)]
- fn enforce_validity(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
- false // for now, we don't enforce validity
- }
-
- #[inline(always)]
- fn enforce_number_init(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
- true
- }
-
- #[inline(always)]
fn checked_binop_checks_overflow(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
true
}
@@ -510,6 +489,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
) -> InterpResult<$tcx, Pointer<Option<AllocId>>> {
// Allow these casts, but make the pointer not dereferenceable.
// (I.e., they behave like transmutation.)
+ // This is correct because no pointers can ever be exposed in compile-time evaluation.
Ok(Pointer::from_addr(addr))
}
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index ed2c4edf9..ed155fbfe 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -21,7 +21,6 @@ use rustc_target::abi::{Align, HasDataLayout, Size};
use super::{
alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, InterpCx,
InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar,
- ScalarMaybeUninit,
};
#[derive(Debug, PartialEq, Copy, Clone)]
@@ -215,7 +214,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.allocate_raw_ptr(alloc, kind).unwrap()
}
- /// This can fail only of `alloc` contains relocations.
+ /// This can fail only of `alloc` contains provenance.
pub fn allocate_raw_ptr(
&mut self,
alloc: Allocation,
@@ -327,7 +326,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Let the machine take some extra action
let size = alloc.size();
- M::memory_deallocated(
+ M::before_memory_deallocation(
*self.tcx,
&mut self.machine,
&mut alloc.extra,
@@ -438,15 +437,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
msg,
})
}
- // Ensure we never consider the null pointer dereferencable.
+ // Ensure we never consider the null pointer dereferenceable.
if M::Provenance::OFFSET_IS_ADDR {
assert_ne!(ptr.addr(), Size::ZERO);
}
// Test align. Check this last; if both bounds and alignment are violated
// we want the error to be about the bounds.
if let Some(align) = align {
- if M::force_int_for_alignment_check(self) {
- // `force_int_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true.
+ if M::use_addr_for_alignment_check(self) {
+ // `use_addr_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true.
check_offset_align(ptr.addr().bytes(), align)?;
} else {
// Check allocation alignment and offset alignment.
@@ -520,6 +519,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Gives raw access to the `Allocation`, without bounds or alignment checks.
/// The caller is responsible for calling the access hooks!
+ ///
+ /// You almost certainly want to use `get_ptr_alloc`/`get_ptr_alloc_mut` instead.
fn get_alloc_raw(
&self,
id: AllocId,
@@ -573,7 +574,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
)?;
if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc {
let range = alloc_range(offset, size);
- M::memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?;
+ M::before_memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?;
Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
} else {
// Even in this branch we have to be sure that we actually access the allocation, in
@@ -589,6 +590,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Ok(&self.get_alloc_raw(id)?.extra)
}
+ /// Return the `mutability` field of the given allocation.
+ pub fn get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability> {
+ Ok(self.get_alloc_raw(id)?.mutability)
+ }
+
/// Gives raw mutable access to the `Allocation`, without bounds or alignment checks.
/// The caller is responsible for calling the access hooks!
///
@@ -634,7 +640,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`.
let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?;
let range = alloc_range(offset, size);
- M::memory_written(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
+ M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id }))
} else {
Ok(None)
@@ -788,10 +794,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
todo.extend(static_roots);
while let Some(id) = todo.pop() {
if reachable.insert(id) {
- // This is a new allocation, add its relocations to `todo`.
+ // This is a new allocation, add the allocation it points to to `todo`.
if let Some((_, alloc)) = self.memory.alloc_map.get(id) {
todo.extend(
- alloc.relocations().values().filter_map(|prov| prov.get_alloc_id()),
+ alloc.provenance().values().filter_map(|prov| prov.get_alloc_id()),
);
}
}
@@ -827,7 +833,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a,
allocs_to_print: &mut VecDeque<AllocId>,
alloc: &Allocation<Prov, Extra>,
) -> std::fmt::Result {
- for alloc_id in alloc.relocations().values().filter_map(|prov| prov.get_alloc_id()) {
+ for alloc_id in alloc.provenance().values().filter_map(|prov| prov.get_alloc_id()) {
allocs_to_print.push_back(alloc_id);
}
write!(fmt, "{}", display_allocation(tcx, alloc))
@@ -894,11 +900,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a,
/// Reading and writing.
impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> {
/// `range` is relative to this allocation reference, not the base of the allocation.
- pub fn write_scalar(
- &mut self,
- range: AllocRange,
- val: ScalarMaybeUninit<Prov>,
- ) -> InterpResult<'tcx> {
+ pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> {
let range = self.range.subrange(range);
debug!("write_scalar at {:?}{range:?}: {val:?}", self.alloc_id);
Ok(self
@@ -908,15 +910,11 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> {
}
/// `offset` is relative to this allocation reference, not the base of the allocation.
- pub fn write_ptr_sized(
- &mut self,
- offset: Size,
- val: ScalarMaybeUninit<Prov>,
- ) -> InterpResult<'tcx> {
+ pub fn write_ptr_sized(&mut self, offset: Size, val: Scalar<Prov>) -> InterpResult<'tcx> {
self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size), val)
}
- /// Mark the entire referenced range as uninitalized
+ /// Mark the entire referenced range as uninitialized
pub fn write_uninit(&mut self) -> InterpResult<'tcx> {
Ok(self
.alloc
@@ -931,7 +929,7 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> {
&self,
range: AllocRange,
read_provenance: bool,
- ) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> {
+ ) -> InterpResult<'tcx, Scalar<Prov>> {
let range = self.range.subrange(range);
let res = self
.alloc
@@ -942,12 +940,12 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> {
}
/// `range` is relative to this allocation reference, not the base of the allocation.
- pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> {
+ pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, Scalar<Prov>> {
self.read_scalar(range, /*read_provenance*/ false)
}
/// `offset` is relative to this allocation reference, not the base of the allocation.
- pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, ScalarMaybeUninit<Prov>> {
+ pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, Scalar<Prov>> {
self.read_scalar(
alloc_range(offset, self.tcx.data_layout().pointer_size),
/*read_provenance*/ true,
@@ -955,29 +953,25 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> {
}
/// `range` is relative to this allocation reference, not the base of the allocation.
- pub fn check_bytes(
- &self,
- range: AllocRange,
- allow_uninit: bool,
- allow_ptr: bool,
- ) -> InterpResult<'tcx> {
+ pub fn get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]> {
Ok(self
.alloc
- .check_bytes(&self.tcx, self.range.subrange(range), allow_uninit, allow_ptr)
+ .get_bytes_strip_provenance(&self.tcx, self.range)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
}
- /// Returns whether the allocation has relocations for the entire range of the `AllocRef`.
- pub(crate) fn has_relocations(&self) -> bool {
- self.alloc.has_relocations(&self.tcx, self.range)
+ /// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`.
+ pub(crate) fn has_provenance(&self) -> bool {
+ self.alloc.range_has_provenance(&self.tcx, self.range)
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
- /// Reads the given number of bytes from memory. Returns them as a slice.
+ /// Reads the given number of bytes from memory, and strips their provenance if possible.
+ /// Returns them as a slice.
///
/// Performs appropriate bounds checks.
- pub fn read_bytes_ptr(
+ pub fn read_bytes_ptr_strip_provenance(
&self,
ptr: Pointer<Option<M::Provenance>>,
size: Size,
@@ -990,7 +984,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// (We are staying inside the bounds here so all is good.)
Ok(alloc_ref
.alloc
- .get_bytes(&alloc_ref.tcx, alloc_ref.range)
+ .get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range)
.map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?)
}
@@ -1071,7 +1065,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
};
let src_alloc = self.get_alloc_raw(src_alloc_id)?;
let src_range = alloc_range(src_offset, size);
- M::memory_read(*tcx, &self.machine, &src_alloc.extra, (src_alloc_id, src_prov), src_range)?;
+ M::before_memory_read(
+ *tcx,
+ &self.machine,
+ &src_alloc.extra,
+ (src_alloc_id, src_prov),
+ src_range,
+ )?;
// We need the `dest` ptr for the next operation, so we get it now.
// We already did the source checks and called the hooks so we are good to return early.
let Some((dest_alloc_id, dest_offset, dest_prov)) = dest_parts else {
@@ -1079,24 +1079,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
return Ok(());
};
- // This checks relocation edges on the src, which needs to happen before
- // `prepare_relocation_copy`.
- let src_bytes = src_alloc
- .get_bytes_with_uninit_and_ptr(&tcx, src_range)
- .map_err(|e| e.to_interp_error(src_alloc_id))?
- .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation
- // first copy the relocations to a temporary buffer, because
- // `get_bytes_mut` will clear the relocations, which is correct,
- // since we don't want to keep any relocations at the target.
- let relocations =
- src_alloc.prepare_relocation_copy(self, src_range, dest_offset, num_copies);
+ // Checks provenance edges on the src, which needs to happen before
+ // `prepare_provenance_copy`.
+ if src_alloc.range_has_provenance(&tcx, alloc_range(src_range.start, Size::ZERO)) {
+ throw_unsup!(PartialPointerCopy(Pointer::new(src_alloc_id, src_range.start)));
+ }
+ if src_alloc.range_has_provenance(&tcx, alloc_range(src_range.end(), Size::ZERO)) {
+ throw_unsup!(PartialPointerCopy(Pointer::new(src_alloc_id, src_range.end())));
+ }
+ let src_bytes = src_alloc.get_bytes_unchecked(src_range).as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation
+ // first copy the provenance to a temporary buffer, because
+ // `get_bytes_mut` will clear the provenance, which is correct,
+ // since we don't want to keep any provenance at the target.
+ let provenance =
+ src_alloc.prepare_provenance_copy(self, src_range, dest_offset, num_copies);
// Prepare a copy of the initialization mask.
let compressed = src_alloc.compress_uninit_range(src_range);
// Destination alloc preparations and access hooks.
let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?;
let dest_range = alloc_range(dest_offset, size * num_copies);
- M::memory_written(
+ M::before_memory_write(
*tcx,
extra,
&mut dest_alloc.extra,
@@ -1118,7 +1121,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
dest_alloc
.write_uninit(&tcx, dest_range)
.map_err(|e| e.to_interp_error(dest_alloc_id))?;
- // We can forget about the relocations, this is all not initialized anyway.
+ // We can forget about the provenance, this is all not initialized anyway.
return Ok(());
}
@@ -1162,8 +1165,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`)
num_copies,
);
- // copy the relocations to the destination
- dest_alloc.mark_relocation_range(relocations);
+ // copy the provenance to the destination
+ dest_alloc.mark_provenance_range(provenance);
Ok(())
}
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 94ba62c16..0e3959b61 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -1,11 +1,9 @@
//! Functions concerning immediate values and operands, and reading from operands.
//! All high-level functions to read from memory work on operands as sources.
-use std::fmt::Write;
-
use rustc_hir::def::Namespace;
use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
-use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter, Printer};
+use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
use rustc_middle::ty::{ConstInt, DelaySpanBugEmitted, Ty};
use rustc_middle::{mir, ty};
use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding};
@@ -14,7 +12,7 @@ use rustc_target::abi::{VariantIdx, Variants};
use super::{
alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId,
InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Place, PlaceTy, Pointer,
- Provenance, Scalar, ScalarMaybeUninit,
+ Provenance, Scalar,
};
/// An `Immediate` represents a single immediate self-contained Rust value.
@@ -27,23 +25,14 @@ use super::{
#[derive(Copy, Clone, Debug)]
pub enum Immediate<Prov: Provenance = AllocId> {
/// A single scalar value (must have *initialized* `Scalar` ABI).
- /// FIXME: we also currently often use this for ZST.
- /// `ScalarMaybeUninit` should reject ZST, and we should use `Uninit` for them instead.
- Scalar(ScalarMaybeUninit<Prov>),
+ Scalar(Scalar<Prov>),
/// A pair of two scalar value (must have `ScalarPair` ABI where both fields are
/// `Scalar::Initialized`).
- ScalarPair(ScalarMaybeUninit<Prov>, ScalarMaybeUninit<Prov>),
+ ScalarPair(Scalar<Prov>, Scalar<Prov>),
/// A value of fully uninitialized memory. Can have and size and layout.
Uninit,
}
-impl<Prov: Provenance> From<ScalarMaybeUninit<Prov>> for Immediate<Prov> {
- #[inline(always)]
- fn from(val: ScalarMaybeUninit<Prov>) -> Self {
- Immediate::Scalar(val)
- }
-}
-
impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
#[inline(always)]
fn from(val: Scalar<Prov>) -> Self {
@@ -51,13 +40,13 @@ impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
}
}
-impl<'tcx, Prov: Provenance> Immediate<Prov> {
+impl<Prov: Provenance> Immediate<Prov> {
pub fn from_pointer(p: Pointer<Prov>, cx: &impl HasDataLayout) -> Self {
- Immediate::Scalar(ScalarMaybeUninit::from_pointer(p, cx))
+ Immediate::Scalar(Scalar::from_pointer(p, cx))
}
pub fn from_maybe_pointer(p: Pointer<Option<Prov>>, cx: &impl HasDataLayout) -> Self {
- Immediate::Scalar(ScalarMaybeUninit::from_maybe_pointer(p, cx))
+ Immediate::Scalar(Scalar::from_maybe_pointer(p, cx))
}
pub fn new_slice(val: Scalar<Prov>, len: u64, cx: &impl HasDataLayout) -> Self {
@@ -69,41 +58,28 @@ impl<'tcx, Prov: Provenance> Immediate<Prov> {
vtable: Pointer<Option<Prov>>,
cx: &impl HasDataLayout,
) -> Self {
- Immediate::ScalarPair(val.into(), ScalarMaybeUninit::from_maybe_pointer(vtable, cx))
+ Immediate::ScalarPair(val.into(), Scalar::from_maybe_pointer(vtable, cx))
}
#[inline]
#[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
- pub fn to_scalar_or_uninit(self) -> ScalarMaybeUninit<Prov> {
+ pub fn to_scalar(self) -> Scalar<Prov> {
match self {
Immediate::Scalar(val) => val,
Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"),
- Immediate::Uninit => ScalarMaybeUninit::Uninit,
+ Immediate::Uninit => bug!("Got uninit where a scalar was expected"),
}
}
#[inline]
#[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
- pub fn to_scalar(self) -> InterpResult<'tcx, Scalar<Prov>> {
- self.to_scalar_or_uninit().check_init()
- }
-
- #[inline]
- #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
- pub fn to_scalar_or_uninit_pair(self) -> (ScalarMaybeUninit<Prov>, ScalarMaybeUninit<Prov>) {
+ pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
match self {
Immediate::ScalarPair(val1, val2) => (val1, val2),
Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"),
- Immediate::Uninit => (ScalarMaybeUninit::Uninit, ScalarMaybeUninit::Uninit),
+ Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
}
}
-
- #[inline]
- #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
- pub fn to_scalar_pair(self) -> InterpResult<'tcx, (Scalar<Prov>, Scalar<Prov>)> {
- let (val1, val2) = self.to_scalar_or_uninit_pair();
- Ok((val1.check_init()?, val2.check_init()?))
- }
}
// ScalarPair needs a type to interpret, so we often have an immediate and a type together
@@ -119,27 +95,17 @@ impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> {
/// Helper function for printing a scalar to a FmtPrinter
fn p<'a, 'tcx, Prov: Provenance>(
cx: FmtPrinter<'a, 'tcx>,
- s: ScalarMaybeUninit<Prov>,
+ s: Scalar<Prov>,
ty: Ty<'tcx>,
) -> Result<FmtPrinter<'a, 'tcx>, std::fmt::Error> {
match s {
- ScalarMaybeUninit::Scalar(Scalar::Int(int)) => {
- cx.pretty_print_const_scalar_int(int, ty, true)
- }
- ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _sz)) => {
+ Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true),
+ Scalar::Ptr(ptr, _sz) => {
// Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to
// print what is points to, which would fail since it has no access to the local
// memory.
cx.pretty_print_const_pointer(ptr, ty, true)
}
- ScalarMaybeUninit::Uninit => cx.typed_value(
- |mut this| {
- this.write_str("uninit ")?;
- Ok(this)
- },
- |this| this.print_type(ty),
- " ",
- ),
}
}
ty::tls::with(|tcx| {
@@ -269,7 +235,7 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
#[inline]
pub fn to_const_int(self) -> ConstInt {
assert!(self.layout.ty.is_integral());
- let int = self.to_scalar().expect("to_const_int doesn't work on scalar pairs").assert_int();
+ let int = self.to_scalar().assert_int();
ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
}
}
@@ -327,7 +293,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn read_immediate_from_mplace_raw(
&self,
mplace: &MPlaceTy<'tcx, M::Provenance>,
- force: bool,
) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
if mplace.layout.is_unsized() {
// Don't touch unsized
@@ -345,47 +310,44 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// case where some of the bytes are initialized and others are not. So, we need an extra
// check that walks over the type of `mplace` to make sure it is truly correct to treat this
// like a `Scalar` (or `ScalarPair`).
- let scalar_layout = match mplace.layout.abi {
- // `if` does not work nested inside patterns, making this a bit awkward to express.
- Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => Some(s),
- Abi::Scalar(s) if force => Some(s.primitive()),
- _ => None,
- };
- if let Some(s) = scalar_layout {
- let size = s.size(self);
- assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
- let scalar = alloc
- .read_scalar(alloc_range(Size::ZERO, size), /*read_provenance*/ s.is_ptr())?;
- return Ok(Some(ImmTy { imm: scalar.into(), layout: mplace.layout }));
- }
- let scalar_pair_layout = match mplace.layout.abi {
+ Ok(match mplace.layout.abi {
+ Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
+ let size = s.size(self);
+ assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
+ let scalar = alloc.read_scalar(
+ alloc_range(Size::ZERO, size),
+ /*read_provenance*/ s.is_ptr(),
+ )?;
+ Some(ImmTy { imm: scalar.into(), layout: mplace.layout })
+ }
Abi::ScalarPair(
abi::Scalar::Initialized { value: a, .. },
abi::Scalar::Initialized { value: b, .. },
- ) => Some((a, b)),
- Abi::ScalarPair(a, b) if force => Some((a.primitive(), b.primitive())),
- _ => None,
- };
- if let Some((a, b)) = scalar_pair_layout {
- // We checked `ptr_align` above, so all fields will have the alignment they need.
- // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
- // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
- let (a_size, b_size) = (a.size(self), b.size(self));
- let b_offset = a_size.align_to(b.align(self).abi);
- assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
- let a_val = alloc.read_scalar(
- alloc_range(Size::ZERO, a_size),
- /*read_provenance*/ a.is_ptr(),
- )?;
- let b_val = alloc
- .read_scalar(alloc_range(b_offset, b_size), /*read_provenance*/ b.is_ptr())?;
- return Ok(Some(ImmTy {
- imm: Immediate::ScalarPair(a_val, b_val),
- layout: mplace.layout,
- }));
- }
- // Neither a scalar nor scalar pair.
- return Ok(None);
+ ) => {
+ // We checked `ptr_align` above, so all fields will have the alignment they need.
+ // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
+ // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
+ let (a_size, b_size) = (a.size(self), b.size(self));
+ let b_offset = a_size.align_to(b.align(self).abi);
+ assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
+ let a_val = alloc.read_scalar(
+ alloc_range(Size::ZERO, a_size),
+ /*read_provenance*/ a.is_ptr(),
+ )?;
+ let b_val = alloc.read_scalar(
+ alloc_range(b_offset, b_size),
+ /*read_provenance*/ b.is_ptr(),
+ )?;
+ Some(ImmTy {
+ imm: Immediate::ScalarPair(a_val.into(), b_val.into()),
+ layout: mplace.layout,
+ })
+ }
+ _ => {
+ // Neither a scalar nor scalar pair.
+ None
+ }
+ })
}
/// Try returning an immediate for the operand. If the layout does not permit loading this as an
@@ -394,20 +356,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// succeed! Whether it succeeds depends on whether the layout can be represented
/// in an `Immediate`, not on which data is stored there currently.
///
- /// If `force` is `true`, then even scalars with fields that can be ununit will be
- /// read. This means the load is lossy and should not be written back!
- /// This flag exists only for validity checking.
- ///
/// This is an internal function that should not usually be used; call `read_immediate` instead.
/// ConstProp needs it, though.
pub fn read_immediate_raw(
&self,
src: &OpTy<'tcx, M::Provenance>,
- force: bool,
) -> InterpResult<'tcx, Result<ImmTy<'tcx, M::Provenance>, MPlaceTy<'tcx, M::Provenance>>> {
Ok(match src.try_as_mplace() {
Ok(ref mplace) => {
- if let Some(val) = self.read_immediate_from_mplace_raw(mplace, force)? {
+ if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
Ok(val)
} else {
Err(*mplace)
@@ -418,24 +375,33 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
/// Read an immediate from a place, asserting that that is possible with the given layout.
+ ///
+ /// If this suceeds, the `ImmTy` is never `Uninit`.
#[inline(always)]
pub fn read_immediate(
&self,
op: &OpTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
- if let Ok(imm) = self.read_immediate_raw(op, /*force*/ false)? {
- Ok(imm)
- } else {
- span_bug!(self.cur_span(), "primitive read failed for type: {:?}", op.layout.ty);
+ if !matches!(
+ op.layout.abi,
+ Abi::Scalar(abi::Scalar::Initialized { .. })
+ | Abi::ScalarPair(abi::Scalar::Initialized { .. }, abi::Scalar::Initialized { .. })
+ ) {
+ span_bug!(self.cur_span(), "primitive read not possible for type: {:?}", op.layout.ty);
}
+ let imm = self.read_immediate_raw(op)?.unwrap();
+ if matches!(*imm, Immediate::Uninit) {
+ throw_ub!(InvalidUninitBytes(None));
+ }
+ Ok(imm)
}
/// Read a scalar from a place
pub fn read_scalar(
&self,
op: &OpTy<'tcx, M::Provenance>,
- ) -> InterpResult<'tcx, ScalarMaybeUninit<M::Provenance>> {
- Ok(self.read_immediate(op)?.to_scalar_or_uninit())
+ ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
+ Ok(self.read_immediate(op)?.to_scalar())
}
/// Read a pointer from a place.
@@ -449,7 +415,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Turn the wide MPlace into a string (must already be dereferenced!)
pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
let len = mplace.len(self)?;
- let bytes = self.read_bytes_ptr(mplace.ptr, Size::from_bytes(len))?;
+ let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len))?;
let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
Ok(str)
}
@@ -478,7 +444,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}
- /// Read from a local. Will not actually access the local if reading from a ZST.
+ /// Read from a local.
/// Will not access memory, instead an indirect `Operand` is returned.
///
/// This is public because it is used by [priroda](https://github.com/oli-obk/priroda) to get an
@@ -490,12 +456,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let layout = self.layout_of_local(frame, local, layout)?;
- let op = if layout.is_zst() {
- // Bypass `access_local` (helps in ConstProp)
- Operand::Immediate(Immediate::Uninit)
- } else {
- *M::access_local(frame, local)?
- };
+ let op = *frame.locals[local].access()?;
Ok(OpTy { op, layout, align: Some(layout.align.abi) })
}
@@ -598,15 +559,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
match c.kind() {
- ty::ConstKind::Param(_) | ty::ConstKind::Bound(..) => throw_inval!(TooGeneric),
+ ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(..) => throw_inval!(TooGeneric),
ty::ConstKind::Error(DelaySpanBugEmitted { reported, .. }) => {
throw_inval!(AlreadyReported(reported))
}
ty::ConstKind::Unevaluated(uv) => {
+ // NOTE: We evaluate to a `ValTree` here as a check to ensure
+ // we're working with valid constants, even though we never need it.
let instance = self.resolve(uv.def, uv.substs)?;
- Ok(self.eval_to_allocation(GlobalId { instance, promoted: uv.promoted })?.into())
+ let cid = GlobalId { instance, promoted: None };
+ let _valtree = self
+ .tcx
+ .eval_to_valtree(self.param_env.and(cid))?
+ .unwrap_or_else(|| bug!("unable to create ValTree for {:?}", uv));
+
+ Ok(self.eval_to_allocation(cid)?.into())
}
- ty::ConstKind::Infer(..) | ty::ConstKind::Placeholder(..) => {
+ ty::ConstKind::Bound(..) | ty::ConstKind::Infer(..) => {
span_bug!(self.cur_span(), "const_to_op: Unexpected ConstKind {:?}", c)
}
ty::ConstKind::Value(valtree) => {
@@ -622,9 +591,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
val: &mir::ConstantKind<'tcx>,
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
+ // FIXME(const_prop): normalization needed b/c const prop lint in
+ // `mir_drops_elaborated_and_const_checked`, which happens before
+ // optimized MIR. Only after optimizing the MIR can we guarantee
+ // that the `RevealAll` pass has happened and that the body's consts
+ // are normalized, so any call to resolve before that needs to be
+ // manually normalized.
+ let val = self.tcx.normalize_erasing_regions(self.param_env, *val);
match val {
- mir::ConstantKind::Ty(ct) => self.const_to_op(*ct, layout),
- mir::ConstantKind::Val(val, ty) => self.const_val_to_op(*val, *ty, layout),
+ mir::ConstantKind::Ty(ct) => self.const_to_op(ct, layout),
+ mir::ConstantKind::Val(val, ty) => self.const_val_to_op(val, ty, layout),
+ mir::ConstantKind::Unevaluated(uv, _) => {
+ let instance = self.resolve(uv.def, uv.substs)?;
+ Ok(self.eval_to_allocation(GlobalId { instance, promoted: uv.promoted })?.into())
+ }
}
}
@@ -727,7 +707,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Figure out which discriminant and variant this corresponds to.
Ok(match *tag_encoding {
TagEncoding::Direct => {
- let scalar = tag_val.to_scalar()?;
+ let scalar = tag_val.to_scalar();
// Generate a specific error if `tag_val` is not an integer.
// (`tag_bits` itself is only used for error messages below.)
let tag_bits = scalar
@@ -757,8 +737,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Return the cast value, and the index.
(discr_val, index.0)
}
- TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
- let tag_val = tag_val.to_scalar()?;
+ TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
+ let tag_val = tag_val.to_scalar();
// Compute the variant this niche value/"tag" corresponds to. With niche layout,
// discriminant (encoded in niche/tag) and variant index are the same.
let variants_start = niche_variants.start().as_u32();
@@ -775,7 +755,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
if !ptr_valid {
throw_ub!(InvalidTag(dbg_val))
}
- dataful_variant
+ untagged_variant
}
Ok(tag_bits) => {
let tag_bits = tag_bits.assert_bits(tag_layout.size);
@@ -785,9 +765,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
let variant_index_relative_val =
self.binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?;
- let variant_index_relative = variant_index_relative_val
- .to_scalar()?
- .assert_bits(tag_val.layout.size);
+ let variant_index_relative =
+ variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size);
// Check if this is in the range that indicates an actual discriminant.
if variant_index_relative <= u128::from(variants_end - variants_start) {
let variant_index_relative = u32::try_from(variant_index_relative)
@@ -806,7 +785,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert!(usize::try_from(variant_index).unwrap() < variants_len);
VariantIdx::from_u32(variant_index)
} else {
- dataful_variant
+ untagged_variant
}
}
};
@@ -820,12 +799,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
mod size_asserts {
use super::*;
+ use rustc_data_structures::static_assert_size;
// These are in alphabetical order, which is easy to maintain.
- rustc_data_structures::static_assert_size!(Immediate, 56);
- rustc_data_structures::static_assert_size!(ImmTy<'_>, 72);
- rustc_data_structures::static_assert_size!(Operand, 64);
- rustc_data_structures::static_assert_size!(OpTy<'_>, 88);
+ static_assert_size!(Immediate, 48);
+ static_assert_size!(ImmTy<'_>, 64);
+ static_assert_size!(Operand, 56);
+ static_assert_size!(OpTy<'_>, 80);
}
diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs
index f9912d706..1f1d06651 100644
--- a/compiler/rustc_const_eval/src/interpret/operator.rs
+++ b/compiler/rustc_const_eval/src/interpret/operator.rs
@@ -329,21 +329,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
match left.layout.ty.kind() {
ty::Char => {
assert_eq!(left.layout.ty, right.layout.ty);
- let left = left.to_scalar()?;
- let right = right.to_scalar()?;
+ let left = left.to_scalar();
+ let right = right.to_scalar();
Ok(self.binary_char_op(bin_op, left.to_char()?, right.to_char()?))
}
ty::Bool => {
assert_eq!(left.layout.ty, right.layout.ty);
- let left = left.to_scalar()?;
- let right = right.to_scalar()?;
+ let left = left.to_scalar();
+ let right = right.to_scalar();
Ok(self.binary_bool_op(bin_op, left.to_bool()?, right.to_bool()?))
}
ty::Float(fty) => {
assert_eq!(left.layout.ty, right.layout.ty);
let ty = left.layout.ty;
- let left = left.to_scalar()?;
- let right = right.to_scalar()?;
+ let left = left.to_scalar();
+ let right = right.to_scalar();
Ok(match fty {
FloatTy::F32 => {
self.binary_float_op(bin_op, ty, left.to_f32()?, right.to_f32()?)
@@ -363,8 +363,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
right.layout.ty
);
- let l = left.to_scalar()?.to_bits(left.layout.size)?;
- let r = right.to_scalar()?.to_bits(right.layout.size)?;
+ let l = left.to_scalar().to_bits(left.layout.size)?;
+ let r = right.to_scalar().to_bits(right.layout.size)?;
self.binary_int_op(bin_op, l, left.layout, r, right.layout)
}
_ if left.layout.ty.is_any_ptr() => {
@@ -410,7 +410,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
use rustc_middle::mir::UnOp::*;
let layout = val.layout;
- let val = val.to_scalar()?;
+ let val = val.to_scalar();
trace!("Running unary op {:?}: {:?} ({:?})", un_op, val, layout.ty);
match layout.ty.kind() {
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index f4571a1ca..b32889290 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -2,8 +2,6 @@
//! into a place.
//! All high-level functions to write to memory work on places as destinations.
-use std::hash::Hash;
-
use rustc_ast::Mutability;
use rustc_middle::mir;
use rustc_middle::ty;
@@ -13,7 +11,7 @@ use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding, Vari
use super::{
alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg,
ConstAlloc, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, OpTy, Operand,
- Pointer, Provenance, Scalar, ScalarMaybeUninit,
+ Pointer, Provenance, Scalar,
};
#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@@ -254,8 +252,6 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
// These are defined here because they produce a place.
impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
#[inline(always)]
- /// Note: do not call `as_ref` on the resulting place. This function should only be used to
- /// read from the resulting mplace, not to get its address back.
pub fn try_as_mplace(&self) -> Result<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
match **self {
Operand::Indirect(mplace) => {
@@ -267,8 +263,6 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
#[inline(always)]
#[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
- /// Note: do not call `as_ref` on the resulting place. This function should only be used to
- /// read from the resulting mplace, not to get its address back.
pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
self.try_as_mplace().unwrap()
}
@@ -294,7 +288,7 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M>
where
- Prov: Provenance + Eq + Hash + 'static,
+ Prov: Provenance + 'static,
M: Machine<'mir, 'tcx, Provenance = Prov>,
{
/// Take a value, which represents a (thin or wide) reference, and make it a place.
@@ -312,7 +306,7 @@ where
let layout = self.layout_of(pointee_type)?;
let (ptr, meta) = match **val {
Immediate::Scalar(ptr) => (ptr, MemPlaceMeta::None),
- Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta.check_init()?)),
+ Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta)),
Immediate::Uninit => throw_ub!(InvalidUninitBytes(None)),
};
@@ -467,7 +461,7 @@ where
#[inline(always)]
pub fn write_scalar(
&mut self,
- val: impl Into<ScalarMaybeUninit<M::Provenance>>,
+ val: impl Into<Scalar<M::Provenance>>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx> {
self.write_immediate(Immediate::Scalar(val.into()), dest)
@@ -644,9 +638,9 @@ where
// Let us see if the layout is simple so we take a shortcut,
// avoid force_allocation.
- let src = match self.read_immediate_raw(src, /*force*/ false)? {
+ let src = match self.read_immediate_raw(src)? {
Ok(src_val) => {
- assert!(!src.layout.is_unsized(), "cannot have unsized immediates");
+ assert!(!src.layout.is_unsized(), "cannot copy unsized immediates");
assert!(
!dest.layout.is_unsized(),
"the src is sized, so the dest must also be sized"
@@ -823,7 +817,7 @@ where
}
abi::Variants::Multiple {
tag_encoding:
- TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+ TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
tag: tag_layout,
tag_field,
..
@@ -831,7 +825,7 @@ where
// No need to validate that the discriminant here because the
// `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
- if variant_index != dataful_variant {
+ if variant_index != untagged_variant {
let variants_start = niche_variants.start().as_u32();
let variant_index_relative = variant_index
.as_u32()
@@ -891,10 +885,13 @@ where
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
mod size_asserts {
use super::*;
+ use rustc_data_structures::static_assert_size;
// These are in alphabetical order, which is easy to maintain.
- rustc_data_structures::static_assert_size!(MemPlaceMeta, 24);
- rustc_data_structures::static_assert_size!(MemPlace, 40);
- rustc_data_structures::static_assert_size!(MPlaceTy<'_>, 64);
- rustc_data_structures::static_assert_size!(Place, 48);
- rustc_data_structures::static_assert_size!(PlaceTy<'_>, 72);
+ static_assert_size!(MemPlaceMeta, 24);
+ static_assert_size!(MemPlace, 40);
+ static_assert_size!(MPlaceTy<'_>, 64);
+ #[cfg(not(bootstrap))]
+ static_assert_size!(Place, 40);
+ #[cfg(not(bootstrap))]
+ static_assert_size!(PlaceTy<'_>, 64);
}
diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs
index 742339f2b..77da8f104 100644
--- a/compiler/rustc_const_eval/src/interpret/projection.rs
+++ b/compiler/rustc_const_eval/src/interpret/projection.rs
@@ -1,14 +1,12 @@
//! This file implements "place projections"; basically a symmetric API for 3 types: MPlaceTy, OpTy, PlaceTy.
//!
-//! OpTy and PlaceTy genrally work by "let's see if we are actually an MPlaceTy, and do something custom if not".
+//! OpTy and PlaceTy generally work by "let's see if we are actually an MPlaceTy, and do something custom if not".
//! For PlaceTy, the custom thing is basically always to call `force_allocation` and then use the MPlaceTy logic anyway.
//! For OpTy, the custom thing on field pojections has to be pretty clever (since `Operand::Immediate` can have fields),
//! but for array/slice operations it only has to worry about `Operand::Uninit`. That makes the value part trivial,
//! but we still need to do bounds checking and adjust the layout. To not duplicate that with MPlaceTy, we actually
//! implement the logic on OpTy, and MPlaceTy calls that.
-use std::hash::Hash;
-
use rustc_middle::mir;
use rustc_middle::ty;
use rustc_middle::ty::layout::LayoutOf;
@@ -22,7 +20,7 @@ use super::{
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M>
where
- Prov: Provenance + Eq + Hash + 'static,
+ Prov: Provenance + 'static,
M: Machine<'mir, 'tcx, Provenance = Prov>,
{
//# Field access
@@ -100,6 +98,8 @@ where
// This makes several assumptions about what layouts we will encounter; we match what
// codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
let field_val: Immediate<_> = match (*base, base.layout.abi) {
+ // if the entire value is uninit, then so is the field (can happen in ConstProp)
+ (Immediate::Uninit, _) => Immediate::Uninit,
// the field contains no information, can be left uninit
_ if field_layout.is_zst() => Immediate::Uninit,
// the field covers the entire type
@@ -124,6 +124,7 @@ where
b_val
})
}
+ // everything else is a bug
_ => span_bug!(
self.cur_span(),
"invalid field access on immediate {}, layout {:#?}",
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index fea158a9f..c6e04cbfb 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -53,7 +53,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.pop_stack_frame(/* unwinding */ true)?;
return Ok(true);
};
- let basic_block = &self.body().basic_blocks()[loc.block];
+ let basic_block = &self.body().basic_blocks[loc.block];
if let Some(stmt) = basic_block.statements.get(loc.statement_index) {
let old_frames = self.frame_idx();
@@ -114,13 +114,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
M::retag(self, *kind, &dest)?;
}
- // Call CopyNonOverlapping
- CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
- let src = self.eval_operand(src, None)?;
- let dst = self.eval_operand(dst, None)?;
- let count = self.eval_operand(count, None)?;
- self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
- }
+ Intrinsic(box ref intrinsic) => self.emulate_nondiverging_intrinsic(intrinsic)?,
// Statements we do not track.
AscribeUserType(..) => {}
@@ -251,8 +245,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Len(place) => {
let src = self.eval_place(place)?;
- let mplace = self.force_allocation(&src)?;
- let len = mplace.len(self)?;
+ let op = self.place_to_op(&src)?;
+ let len = op.len(self)?;
self.write_scalar(Scalar::from_machine_usize(len, self), &dest)?;
}
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index d563e35f9..50a82aa0e 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -1,5 +1,6 @@
use std::borrow::Cow;
+use rustc_ast::ast::InlineAsmOptions;
use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
use rustc_middle::ty::Instance;
use rustc_middle::{
@@ -129,8 +130,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
Assert { ref cond, expected, ref msg, target, cleanup } => {
- let cond_val =
- self.read_immediate(&self.eval_operand(cond, None)?)?.to_scalar()?.to_bool()?;
+ let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?;
if expected == cond_val {
self.go_to_block(target);
} else {
@@ -167,8 +167,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
terminator.kind
),
- // Inline assembly can't be interpreted.
- InlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"),
+ InlineAsm { template, ref operands, options, destination, .. } => {
+ M::eval_inline_asm(self, template, operands, options)?;
+ if options.contains(InlineAsmOptions::NORETURN) {
+ throw_ub_format!("returned from noreturn inline assembly");
+ }
+ self.go_to_block(
+ destination
+ .expect("InlineAsm terminators without noreturn must have a destination"),
+ )
+ }
}
Ok(())
@@ -215,12 +223,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
_ => false,
}
};
- // Padding must be fully equal.
- let pad_compat = || caller_abi.pad == callee_abi.pad;
// When comparing the PassMode, we have to be smart about comparing the attributes.
- let arg_attr_compat = |a1: ArgAttributes, a2: ArgAttributes| {
+ let arg_attr_compat = |a1: &ArgAttributes, a2: &ArgAttributes| {
// There's only one regular attribute that matters for the call ABI: InReg.
- // Everything else is things like noalias, dereferencable, nonnull, ...
+ // Everything else is things like noalias, dereferenceable, nonnull, ...
// (This also applies to pointee_size, pointee_align.)
if a1.regular.contains(ArgAttribute::InReg) != a2.regular.contains(ArgAttribute::InReg)
{
@@ -233,13 +239,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
return true;
};
- let mode_compat = || match (caller_abi.mode, callee_abi.mode) {
+ let mode_compat = || match (&caller_abi.mode, &callee_abi.mode) {
(PassMode::Ignore, PassMode::Ignore) => true,
(PassMode::Direct(a1), PassMode::Direct(a2)) => arg_attr_compat(a1, a2),
(PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => {
arg_attr_compat(a1, a2) && arg_attr_compat(b1, b2)
}
- (PassMode::Cast(c1), PassMode::Cast(c2)) => c1 == c2,
+ (PassMode::Cast(c1, pad1), PassMode::Cast(c2, pad2)) => c1 == c2 && pad1 == pad2,
(
PassMode::Indirect { attrs: a1, extra_attrs: None, on_stack: s1 },
PassMode::Indirect { attrs: a2, extra_attrs: None, on_stack: s2 },
@@ -251,7 +257,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
_ => false,
};
- if layout_compat() && pad_compat() && mode_compat() {
+ if layout_compat() && mode_compat() {
return true;
}
trace!(
@@ -534,7 +540,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let mut non_zst_field = None;
for i in 0..receiver.layout.fields.count() {
let field = self.operand_field(&receiver, i)?;
- if !field.layout.is_zst() {
+ let zst =
+ field.layout.is_zst() && field.layout.align.abi.bytes() == 1;
+ if !zst {
assert!(
non_zst_field.is_none(),
"multiple non-ZST fields in dyn receiver type {}",
@@ -557,7 +565,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
.tcx
.struct_tail_erasing_lifetimes(receiver_place.layout.ty, self.param_env);
let ty::Dynamic(data, ..) = receiver_tail.kind() else {
- span_bug!(self.cur_span(), "dyanmic call on non-`dyn` type {}", receiver_tail)
+ span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail)
};
// Get the required information from the vtable.
diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs
index b3a511d5a..cab23b724 100644
--- a/compiler/rustc_const_eval/src/interpret/traits.rs
+++ b/compiler/rustc_const_eval/src/interpret/traits.rs
@@ -32,7 +32,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Ok(vtable_ptr.into())
}
- /// Returns a high-level representation of the entires of the given vtable.
+ /// Returns a high-level representation of the entries of the given vtable.
pub fn get_vtable_entries(
&self,
vtable: Pointer<Option<M::Provenance>>,
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 0e50d1ed4..14aaee6ac 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -5,9 +5,10 @@
//! to be const-safe.
use std::convert::TryFrom;
-use std::fmt::Write;
+use std::fmt::{Display, Write};
use std::num::NonZeroUsize;
+use rustc_ast::Mutability;
use rustc_data_structures::fx::FxHashSet;
use rustc_hir as hir;
use rustc_middle::mir::interpret::InterpError;
@@ -19,9 +20,11 @@ use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, VariantIdx, Variants, Wr
use std::hash::Hash;
+// for the validation errors
+use super::UndefinedBehaviorInfo::*;
use super::{
- alloc_range, CheckInAllocMsg, GlobalAlloc, Immediate, InterpCx, InterpResult, MPlaceTy,
- Machine, MemPlaceMeta, OpTy, Scalar, ScalarMaybeUninit, ValueVisitor,
+ CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine,
+ MemPlaceMeta, OpTy, Scalar, ValueVisitor,
};
macro_rules! throw_validation_failure {
@@ -59,6 +62,7 @@ macro_rules! throw_validation_failure {
/// });
/// ```
///
+/// The patterns must be of type `UndefinedBehaviorInfo`.
/// An additional expected parameter can also be added to the failure message:
///
/// ```
@@ -86,7 +90,7 @@ macro_rules! try_validation {
// allocation here as this can only slow down builds that fail anyway.
Err(e) => match e.kind() {
$(
- $($p)|+ =>
+ InterpError::UndefinedBehavior($($p)|+) =>
throw_validation_failure!(
$where,
{ $( $what_fmt ),+ } $( expected { $( $expected_fmt ),+ } )?
@@ -304,6 +308,26 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
Ok(r)
}
+ fn read_immediate(
+ &self,
+ op: &OpTy<'tcx, M::Provenance>,
+ expected: impl Display,
+ ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
+ Ok(try_validation!(
+ self.ecx.read_immediate(op),
+ self.path,
+ InvalidUninitBytes(None) => { "uninitialized memory" } expected { "{expected}" }
+ ))
+ }
+
+ fn read_scalar(
+ &self,
+ op: &OpTy<'tcx, M::Provenance>,
+ expected: impl Display,
+ ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
+ Ok(self.read_immediate(op, expected)?.to_scalar())
+ }
+
fn check_wide_ptr_meta(
&mut self,
meta: MemPlaceMeta<M::Provenance>,
@@ -317,8 +341,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
let (_ty, _trait) = try_validation!(
self.ecx.get_ptr_vtable(vtable),
self.path,
- err_ub!(DanglingIntPointer(..)) |
- err_ub!(InvalidVTablePointer(..)) =>
+ DanglingIntPointer(..) |
+ InvalidVTablePointer(..) =>
{ "{vtable}" } expected { "a vtable pointer" },
);
// FIXME: check if the type/trait match what ty::Dynamic says?
@@ -344,14 +368,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
value: &OpTy<'tcx, M::Provenance>,
kind: &str,
) -> InterpResult<'tcx> {
- let value = self.ecx.read_immediate(value)?;
+ let place =
+ self.ecx.ref_to_mplace(&self.read_immediate(value, format_args!("a {kind}"))?)?;
// Handle wide pointers.
// Check metadata early, for better diagnostics
- let place = try_validation!(
- self.ecx.ref_to_mplace(&value),
- self.path,
- err_ub!(InvalidUninitBytes(None)) => { "uninitialized {}", kind },
- );
if place.layout.is_unsized() {
self.check_wide_ptr_meta(place.meta, place.layout)?;
}
@@ -359,7 +379,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
let size_and_align = try_validation!(
self.ecx.size_and_align_of_mplace(&place),
self.path,
- err_ub!(InvalidMeta(msg)) => { "invalid {} metadata: {}", kind, msg },
+ InvalidMeta(msg) => { "invalid {} metadata: {}", kind, msg },
);
let (size, align) = size_and_align
// for the purpose of validity, consider foreign types to have
@@ -375,21 +395,21 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message
),
self.path,
- err_ub!(AlignmentCheckFailed { required, has }) =>
+ AlignmentCheckFailed { required, has } =>
{
"an unaligned {kind} (required {} byte alignment but found {})",
required.bytes(),
has.bytes()
},
- err_ub!(DanglingIntPointer(0, _)) =>
+ DanglingIntPointer(0, _) =>
{ "a null {kind}" },
- err_ub!(DanglingIntPointer(i, _)) =>
+ DanglingIntPointer(i, _) =>
{ "a dangling {kind} (address {i:#x} is unallocated)" },
- err_ub!(PointerOutOfBounds { .. }) =>
+ PointerOutOfBounds { .. } =>
{ "a dangling {kind} (going beyond the bounds of its allocation)" },
// This cannot happen during const-eval (because interning already detects
// dangling pointers), but it can happen in Miri.
- err_ub!(PointerUseAfterFree(..)) =>
+ PointerUseAfterFree(..) =>
{ "a dangling {kind} (use-after-free)" },
);
// Do not allow pointers to uninhabited types.
@@ -403,34 +423,51 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
// Proceed recursively even for ZST, no reason to skip them!
// `!` is a ZST and we want to validate it.
if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr) {
- // Special handling for pointers to statics (irrespective of their type).
+ // Let's see what kind of memory this points to.
let alloc_kind = self.ecx.tcx.try_get_global_alloc(alloc_id);
- if let Some(GlobalAlloc::Static(did)) = alloc_kind {
- assert!(!self.ecx.tcx.is_thread_local_static(did));
- assert!(self.ecx.tcx.is_static(did));
- if matches!(
- self.ctfe_mode,
- Some(CtfeValidationMode::Const { allow_static_ptrs: false, .. })
- ) {
- // See const_eval::machine::MemoryExtra::can_access_statics for why
- // this check is so important.
- // This check is reachable when the const just referenced the static,
- // but never read it (so we never entered `before_access_global`).
- throw_validation_failure!(self.path,
- { "a {} pointing to a static variable", kind }
- );
+ match alloc_kind {
+ Some(GlobalAlloc::Static(did)) => {
+ // Special handling for pointers to statics (irrespective of their type).
+ assert!(!self.ecx.tcx.is_thread_local_static(did));
+ assert!(self.ecx.tcx.is_static(did));
+ if matches!(
+ self.ctfe_mode,
+ Some(CtfeValidationMode::Const { allow_static_ptrs: false, .. })
+ ) {
+ // See const_eval::machine::MemoryExtra::can_access_statics for why
+ // this check is so important.
+ // This check is reachable when the const just referenced the static,
+ // but never read it (so we never entered `before_access_global`).
+ throw_validation_failure!(self.path,
+ { "a {} pointing to a static variable in a constant", kind }
+ );
+ }
+ // We skip recursively checking other statics. These statics must be sound by
+ // themselves, and the only way to get broken statics here is by using
+ // unsafe code.
+ // The reasons we don't check other statics is twofold. For one, in all
+ // sound cases, the static was already validated on its own, and second, we
+ // trigger cycle errors if we try to compute the value of the other static
+ // and that static refers back to us.
+ // We might miss const-invalid data,
+ // but things are still sound otherwise (in particular re: consts
+ // referring to statics).
+ return Ok(());
}
- // We skip checking other statics. These statics must be sound by
- // themselves, and the only way to get broken statics here is by using
- // unsafe code.
- // The reasons we don't check other statics is twofold. For one, in all
- // sound cases, the static was already validated on its own, and second, we
- // trigger cycle errors if we try to compute the value of the other static
- // and that static refers back to us.
- // We might miss const-invalid data,
- // but things are still sound otherwise (in particular re: consts
- // referring to statics).
- return Ok(());
+ Some(GlobalAlloc::Memory(alloc)) => {
+ if alloc.inner().mutability == Mutability::Mut
+ && matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. }))
+ {
+ // This should be unreachable, but if someone manages to copy a pointer
+ // out of a `static`, then that pointer might point to mutable memory,
+ // and we would catch that here.
+ throw_validation_failure!(self.path,
+ { "a {} pointing to mutable memory in a constant", kind }
+ );
+ }
+ }
+ // Nothing to check for these.
+ None | Some(GlobalAlloc::Function(..) | GlobalAlloc::VTable(..)) => {}
}
}
let path = &self.path;
@@ -446,20 +483,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
Ok(())
}
- fn read_scalar(
- &self,
- op: &OpTy<'tcx, M::Provenance>,
- ) -> InterpResult<'tcx, ScalarMaybeUninit<M::Provenance>> {
- self.ecx.read_scalar(op)
- }
-
- fn read_immediate_forced(
- &self,
- op: &OpTy<'tcx, M::Provenance>,
- ) -> InterpResult<'tcx, Immediate<M::Provenance>> {
- Ok(*self.ecx.read_immediate_raw(op, /*force*/ true)?.unwrap())
- }
-
/// Check if this is a value of primitive type, and if yes check the validity of the value
/// at that type. Return `true` if the type is indeed primitive.
fn try_visit_primitive(
@@ -470,41 +493,39 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
let ty = value.layout.ty;
match ty.kind() {
ty::Bool => {
- let value = self.read_scalar(value)?;
+ let value = self.read_scalar(value, "a boolean")?;
try_validation!(
value.to_bool(),
self.path,
- err_ub!(InvalidBool(..)) | err_ub!(InvalidUninitBytes(None)) =>
+ InvalidBool(..) =>
{ "{:x}", value } expected { "a boolean" },
);
Ok(true)
}
ty::Char => {
- let value = self.read_scalar(value)?;
+ let value = self.read_scalar(value, "a unicode scalar value")?;
try_validation!(
value.to_char(),
self.path,
- err_ub!(InvalidChar(..)) | err_ub!(InvalidUninitBytes(None)) =>
+ InvalidChar(..) =>
{ "{:x}", value } expected { "a valid unicode scalar value (in `0..=0x10FFFF` but not in `0xD800..=0xDFFF`)" },
);
Ok(true)
}
ty::Float(_) | ty::Int(_) | ty::Uint(_) => {
- let value = self.read_scalar(value)?;
// NOTE: Keep this in sync with the array optimization for int/float
// types below!
- if M::enforce_number_init(self.ecx) {
- try_validation!(
- value.check_init(),
- self.path,
- err_ub!(InvalidUninitBytes(..)) =>
- { "{:x}", value } expected { "initialized bytes" }
- );
- }
+ let value = self.read_scalar(
+ value,
+ if matches!(ty.kind(), ty::Float(..)) {
+ "a floating point number"
+ } else {
+ "an integer"
+ },
+ )?;
// As a special exception we *do* match on a `Scalar` here, since we truly want
// to know its underlying representation (and *not* cast it to an integer).
- let is_ptr = value.check_init().map_or(false, |v| matches!(v, Scalar::Ptr(..)));
- if is_ptr {
+ if matches!(value, Scalar::Ptr(..)) {
throw_validation_failure!(self.path,
{ "{:x}", value } expected { "plain (non-pointer) bytes" }
)
@@ -515,11 +536,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
// We are conservative with uninit for integers, but try to
// actually enforce the strict rules for raw pointers (mostly because
// that lets us re-use `ref_to_mplace`).
- let place = try_validation!(
- self.ecx.read_immediate(value).and_then(|ref i| self.ecx.ref_to_mplace(i)),
- self.path,
- err_ub!(InvalidUninitBytes(None)) => { "uninitialized raw pointer" },
- );
+ let place =
+ self.ecx.ref_to_mplace(&self.read_immediate(value, "a raw pointer")?)?;
if place.layout.is_unsized() {
self.check_wide_ptr_meta(place.meta, place.layout)?;
}
@@ -527,7 +545,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
}
ty::Ref(_, ty, mutbl) => {
if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. }))
- && *mutbl == hir::Mutability::Mut
+ && *mutbl == Mutability::Mut
{
// A mutable reference inside a const? That does not seem right (except if it is
// a ZST).
@@ -540,11 +558,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
Ok(true)
}
ty::FnPtr(_sig) => {
- let value = try_validation!(
- self.ecx.read_scalar(value).and_then(|v| v.check_init()),
- self.path,
- err_ub!(InvalidUninitBytes(None)) => { "uninitialized bytes" } expected { "a proper pointer or integer value" },
- );
+ let value = self.read_scalar(value, "a function pointer")?;
// If we check references recursively, also check that this points to a function.
if let Some(_) = self.ref_tracking {
@@ -552,8 +566,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
let _fn = try_validation!(
self.ecx.get_ptr_fn(ptr),
self.path,
- err_ub!(DanglingIntPointer(..)) |
- err_ub!(InvalidFunctionPointer(..)) =>
+ DanglingIntPointer(..) |
+ InvalidFunctionPointer(..) =>
{ "{ptr}" } expected { "a function pointer" },
);
// FIXME: Check if the signature matches
@@ -595,40 +609,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
fn visit_scalar(
&mut self,
- scalar: ScalarMaybeUninit<M::Provenance>,
+ scalar: Scalar<M::Provenance>,
scalar_layout: ScalarAbi,
) -> InterpResult<'tcx> {
- // We check `is_full_range` in a slightly complicated way because *if* we are checking
- // number validity, then we want to ensure that `Scalar::Initialized` is indeed initialized,
- // i.e. that we go over the `check_init` below.
let size = scalar_layout.size(self.ecx);
- let is_full_range = match scalar_layout {
- ScalarAbi::Initialized { .. } => {
- if M::enforce_number_init(self.ecx) {
- false // not "full" since uninit is not accepted
- } else {
- scalar_layout.is_always_valid(self.ecx)
- }
- }
- ScalarAbi::Union { .. } => true,
- };
- if is_full_range {
- // Nothing to check. Cruciall we don't even `read_scalar` until here, since that would
- // fail for `Union` scalars!
- return Ok(());
- }
- // We have something to check: it must at least be initialized.
let valid_range = scalar_layout.valid_range(self.ecx);
let WrappingRange { start, end } = valid_range;
let max_value = size.unsigned_int_max();
assert!(end <= max_value);
- let value = try_validation!(
- scalar.check_init(),
- self.path,
- err_ub!(InvalidUninitBytes(None)) => { "{:x}", scalar }
- expected { "something {}", wrapping_range_format(valid_range, max_value) },
- );
- let bits = match value.try_to_int() {
+ let bits = match scalar.try_to_int() {
Ok(int) => int.assert_bits(size),
Err(_) => {
// So this is a pointer then, and casting to an int failed.
@@ -636,7 +625,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
// We support 2 kinds of ranges here: full range, and excluding zero.
if start == 1 && end == max_value {
// Only null is the niche. So make sure the ptr is NOT null.
- if self.ecx.scalar_may_be_null(value)? {
+ if self.ecx.scalar_may_be_null(scalar)? {
throw_validation_failure!(self.path,
{ "a potentially null pointer" }
expected {
@@ -693,9 +682,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
Ok(try_validation!(
this.ecx.read_discriminant(op),
this.path,
- err_ub!(InvalidTag(val)) =>
+ InvalidTag(val) =>
{ "{:x}", val } expected { "a valid enum tag" },
- err_ub!(InvalidUninitBytes(None)) =>
+ InvalidUninitBytes(None) =>
{ "uninitialized bytes" } expected { "a valid enum tag" },
)
.1)
@@ -788,10 +777,11 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
);
}
Abi::Scalar(scalar_layout) => {
- // We use a 'forced' read because we always need a `Immediate` here
- // and treating "partially uninit" as "fully uninit" is fine for us.
- let scalar = self.read_immediate_forced(op)?.to_scalar_or_uninit();
- self.visit_scalar(scalar, scalar_layout)?;
+ if !scalar_layout.is_uninit_valid() {
+ // There is something to check here.
+ let scalar = self.read_scalar(op, "initiailized scalar value")?;
+ self.visit_scalar(scalar, scalar_layout)?;
+ }
}
Abi::ScalarPair(a_layout, b_layout) => {
// There is no `rustc_layout_scalar_valid_range_start` for pairs, so
@@ -799,10 +789,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
// but that can miss bugs in layout computation. Layout computation
// is subtle due to enums having ScalarPair layout, where one field
// is the discriminant.
- if cfg!(debug_assertions) {
- // We use a 'forced' read because we always need a `Immediate` here
- // and treating "partially uninit" as "fully uninit" is fine for us.
- let (a, b) = self.read_immediate_forced(op)?.to_scalar_or_uninit_pair();
+ if cfg!(debug_assertions)
+ && !a_layout.is_uninit_valid()
+ && !b_layout.is_uninit_valid()
+ {
+ // We can only proceed if *both* scalars need to be initialized.
+ // FIXME: find a way to also check ScalarPair when one side can be uninit but
+ // the other must be init.
+ let (a, b) =
+ self.read_immediate(op, "initiailized scalar value")?.to_scalar_pair();
self.visit_scalar(a, a_layout)?;
self.visit_scalar(b, b_layout)?;
}
@@ -830,9 +825,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate
let len = mplace.len(self.ecx)?;
try_validation!(
- self.ecx.read_bytes_ptr(mplace.ptr, Size::from_bytes(len)),
+ self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len)),
self.path,
- err_ub!(InvalidUninitBytes(..)) => { "uninitialized data in `str`" },
+ InvalidUninitBytes(..) => { "uninitialized data in `str`" },
);
}
ty::Array(tys, ..) | ty::Slice(tys)
@@ -880,13 +875,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
// We also accept uninit, for consistency with the slow path.
let alloc = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)?.expect("we already excluded size 0");
- match alloc.check_bytes(
- alloc_range(Size::ZERO, size),
- /*allow_uninit*/ !M::enforce_number_init(self.ecx),
- /*allow_ptr*/ false,
- ) {
+ match alloc.get_bytes_strip_provenance() {
// In the happy case, we needn't check anything else.
- Ok(()) => {}
+ Ok(_) => {}
// Some error happened, try to provide a more detailed description.
Err(err) => {
// For some errors we might be able to provide extra information.
@@ -981,6 +972,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// It will error if the bits at the destination do not match the ones described by the layout.
#[inline(always)]
pub fn validate_operand(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
+ // Note that we *could* actually be in CTFE here with `-Zextra-const-ub-checks`, but it's
+ // still correct to not use `ctfe_mode`: that mode is for validation of the final constant
+ // value, it rules out things like `UnsafeCell` in awkward places. It also can make checking
+ // recurse through references which, for now, we don't want here, either.
self.validate_operand_internal(op, vec![], None, None)
}
}
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index 72ac6af68..9f47d302a 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -10,7 +10,7 @@ Rust MIR: a lowered representation of Rust.
#![feature(decl_macro)]
#![feature(exact_size_is_empty)]
#![feature(let_chains)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
#![feature(map_try_insert)]
#![feature(min_specialization)]
#![feature(slice_ptr_get)]
diff --git a/compiler/rustc_const_eval/src/might_permit_raw_init.rs b/compiler/rustc_const_eval/src/might_permit_raw_init.rs
index f971c2238..37ffa19cc 100644
--- a/compiler/rustc_const_eval/src/might_permit_raw_init.rs
+++ b/compiler/rustc_const_eval/src/might_permit_raw_init.rs
@@ -13,7 +13,11 @@ pub fn might_permit_raw_init<'tcx>(
let strict = tcx.sess.opts.unstable_opts.strict_init_checks;
if strict {
- let machine = CompileTimeInterpreter::new(Limit::new(0), false);
+ let machine = CompileTimeInterpreter::new(
+ Limit::new(0),
+ /*can_access_statics:*/ false,
+ /*check_alignment:*/ true,
+ );
let mut cx = InterpCx::new(tcx, rustc_span::DUMMY_SP, ParamEnv::reveal_all(), machine);
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index 0adb88a18..7e15858c8 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -135,7 +135,7 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
// qualifs for the return type.
let return_block = ccx
.body
- .basic_blocks()
+ .basic_blocks
.iter_enumerated()
.find(|(_, block)| matches!(block.terminator().kind, TerminatorKind::Return))
.map(|(bb, _)| bb);
@@ -546,6 +546,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
// Since no pointer can ever get exposed (rejected above), this is easy to support.
}
+ Rvalue::Cast(CastKind::DynStar, _, _) => {
+ unimplemented!()
+ }
+
Rvalue::Cast(CastKind::Misc, _, _) => {}
Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => {}
@@ -678,7 +682,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
| StatementKind::Retag { .. }
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
- | StatementKind::CopyNonOverlapping(..)
+ | StatementKind::Intrinsic(..)
| StatementKind::Nop => {}
}
}
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
index 338022616..5fb4bf638 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
@@ -1,6 +1,7 @@
//! Concrete error types for all operations which may be invalid in a certain const context.
use hir::def_id::LocalDefId;
+use hir::ConstContext;
use rustc_errors::{
error_code, struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed,
};
@@ -23,8 +24,11 @@ use rustc_trait_selection::traits::SelectionContext;
use super::ConstCx;
use crate::errors::{
- MutDerefErr, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr,
- TransientMutBorrowErr, TransientMutBorrowErrRaw,
+ InteriorMutabilityBorrow, InteriorMutableDataRefer, MutDerefErr, NonConstFmtMacroCall,
+ NonConstFnCall, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr,
+ TransientMutBorrowErr, TransientMutBorrowErrRaw, UnallowedFnPointerCall,
+ UnallowedHeapAllocations, UnallowedInlineAsm, UnallowedMutableRefs, UnallowedMutableRefsRaw,
+ UnallowedOpInConstContext, UnstableConstFn,
};
use crate::util::{call_kind, CallDesugaringKind, CallKind};
@@ -96,10 +100,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallIndirect {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- ccx.tcx.sess.struct_span_err(
- span,
- &format!("function pointer calls are not allowed in {}s", ccx.const_kind()),
- )
+ ccx.tcx.sess.create_err(UnallowedFnPointerCall { span, kind: ccx.const_kind() })
}
}
@@ -307,22 +308,13 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
err
}
_ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::ArgumentV1Methods) => {
- struct_span_err!(
- ccx.tcx.sess,
- span,
- E0015,
- "cannot call non-const formatting macro in {}s",
- ccx.const_kind(),
- )
+ ccx.tcx.sess.create_err(NonConstFmtMacroCall { span, kind: ccx.const_kind() })
}
- _ => struct_span_err!(
- ccx.tcx.sess,
+ _ => ccx.tcx.sess.create_err(NonConstFnCall {
span,
- E0015,
- "cannot call non-const fn `{}` in {}s",
- ccx.tcx.def_path_str_with_substs(callee, substs),
- ccx.const_kind(),
- ),
+ def_path_str: ccx.tcx.def_path_str_with_substs(callee, substs),
+ kind: ccx.const_kind(),
+ }),
};
err.note(&format!(
@@ -331,6 +323,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
ccx.const_kind(),
));
+ if let ConstContext::Static(_) = ccx.const_kind() {
+ err.note("consider wrapping this expression in `Lazy::new(|| ...)` from the `once_cell` crate: https://crates.io/crates/once_cell");
+ }
+
err
}
}
@@ -349,10 +345,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallUnstable {
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
let FnCallUnstable(def_id, feature) = *self;
- let mut err = ccx.tcx.sess.struct_span_err(
- span,
- &format!("`{}` is not yet stable as a const fn", ccx.tcx.def_path_str(def_id)),
- );
+ let mut err = ccx
+ .tcx
+ .sess
+ .create_err(UnstableConstFn { span, def_path: ccx.tcx.def_path_str(def_id) });
if ccx.is_const_stable_const_fn() {
err.help("const-stable functions can only call other const-stable functions");
@@ -387,9 +383,12 @@ impl<'tcx> NonConstOp<'tcx> for Generator {
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
let msg = format!("{}s are not allowed in {}s", self.0, ccx.const_kind());
if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 {
- feature_err(&ccx.tcx.sess.parse_sess, sym::const_async_blocks, span, &msg)
+ ccx.tcx.sess.create_feature_err(
+ UnallowedOpInConstContext { span, msg },
+ sym::const_async_blocks,
+ )
} else {
- ccx.tcx.sess.struct_span_err(span, &msg)
+ ccx.tcx.sess.create_err(UnallowedOpInConstContext { span, msg })
}
}
}
@@ -402,23 +401,11 @@ impl<'tcx> NonConstOp<'tcx> for HeapAllocation {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- let mut err = struct_span_err!(
- ccx.tcx.sess,
+ ccx.tcx.sess.create_err(UnallowedHeapAllocations {
span,
- E0010,
- "allocations are not allowed in {}s",
- ccx.const_kind()
- );
- err.span_label(span, format!("allocation not allowed in {}s", ccx.const_kind()));
- if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
- err.note(
- "The value of statics and constants must be known at compile time, \
- and they live for the entire lifetime of a program. Creating a boxed \
- value allocates memory on the heap at runtime, and therefore cannot \
- be done at compile time.",
- );
- }
- err
+ kind: ccx.const_kind(),
+ teach: ccx.tcx.sess.teach(&error_code!(E0010)).then_some(()),
+ })
}
}
@@ -430,13 +417,7 @@ impl<'tcx> NonConstOp<'tcx> for InlineAsm {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- struct_span_err!(
- ccx.tcx.sess,
- span,
- E0015,
- "inline assembly is not allowed in {}s",
- ccx.const_kind()
- )
+ ccx.tcx.sess.create_err(UnallowedInlineAsm { span, kind: ccx.const_kind() })
}
}
@@ -482,12 +463,7 @@ impl<'tcx> NonConstOp<'tcx> for TransientCellBorrow {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- feature_err(
- &ccx.tcx.sess.parse_sess,
- sym::const_refs_to_cell,
- span,
- "cannot borrow here, since the borrowed element may contain interior mutability",
- )
+ ccx.tcx.sess.create_feature_err(InteriorMutabilityBorrow { span }, sym::const_refs_to_cell)
}
}
@@ -502,32 +478,22 @@ impl<'tcx> NonConstOp<'tcx> for CellBorrow {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- let mut err = struct_span_err!(
- ccx.tcx.sess,
- span,
- E0492,
- "{}s cannot refer to interior mutable data",
- ccx.const_kind(),
- );
- err.span_label(
- span,
- "this borrow of an interior mutable value may end up in the final value",
- );
+ // FIXME: Maybe a more elegant solution to this if else case
if let hir::ConstContext::Static(_) = ccx.const_kind() {
- err.help(
- "to fix this, the value can be extracted to a separate \
- `static` item and then referenced",
- );
- }
- if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
- err.note(
- "A constant containing interior mutable data behind a reference can allow you
- to modify that data. This would make multiple uses of a constant to be able to
- see different values and allow circumventing the `Send` and `Sync` requirements
- for shared mutable data, which is unsound.",
- );
+ ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+ span,
+ opt_help: Some(()),
+ kind: ccx.const_kind(),
+ teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()),
+ })
+ } else {
+ ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+ span,
+ opt_help: None,
+ kind: ccx.const_kind(),
+ teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()),
+ })
}
- err
}
}
@@ -553,33 +519,18 @@ impl<'tcx> NonConstOp<'tcx> for MutBorrow {
ccx: &ConstCx<'_, 'tcx>,
span: Span,
) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
- let raw = match self.0 {
- hir::BorrowKind::Raw => "raw ",
- hir::BorrowKind::Ref => "",
- };
-
- let mut err = struct_span_err!(
- ccx.tcx.sess,
- span,
- E0764,
- "{}mutable references are not allowed in the final value of {}s",
- raw,
- ccx.const_kind(),
- );
-
- if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
- err.note(
- "References in statics and constants may only refer \
- to immutable values.\n\n\
- Statics are shared everywhere, and if they refer to \
- mutable data one might violate memory safety since \
- holding multiple mutable references to shared data \
- is not allowed.\n\n\
- If you really want global mutable state, try using \
- static mut or a global UnsafeCell.",
- );
+ match self.0 {
+ hir::BorrowKind::Raw => ccx.tcx.sess.create_err(UnallowedMutableRefsRaw {
+ span,
+ kind: ccx.const_kind(),
+ teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
+ }),
+ hir::BorrowKind::Ref => ccx.tcx.sess.create_err(UnallowedMutableRefs {
+ span,
+ kind: ccx.const_kind(),
+ teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
+ }),
}
- err
}
}
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index c8a63c9c3..6c73ef5a8 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -5,12 +5,11 @@
use rustc_errors::ErrorGuaranteed;
use rustc_hir::LangItem;
use rustc_infer::infer::TyCtxtInferExt;
-use rustc_infer::traits::TraitEngine;
use rustc_middle::mir::*;
use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty};
use rustc_span::DUMMY_SP;
use rustc_trait_selection::traits::{
- self, ImplSource, Obligation, ObligationCause, SelectionContext, TraitEngineExt,
+ self, ImplSource, Obligation, ObligationCause, SelectionContext,
};
use super::ConstCx;
@@ -189,15 +188,8 @@ impl Qualif for NeedsNonConstDrop {
return false;
}
- // If we successfully found one, then select all of the predicates
- // implied by our const drop impl.
- let mut fcx = <dyn TraitEngine<'tcx>>::new(cx.tcx);
- for nested in impl_src.nested_obligations() {
- fcx.register_predicate_obligation(&infcx, nested);
- }
-
// If we had any errors, then it's bad
- !fcx.select_all_or_error(&infcx).is_empty()
+ !traits::fully_solve_obligations(&infcx, impl_src.nested_obligations()).is_empty()
})
}
@@ -354,31 +346,43 @@ where
};
// Check the qualifs of the value of `const` items.
- if let Some(ct) = constant.literal.const_for_ty() {
- if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted }) = ct.kind()
- {
- // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible
- // only for `NeedsNonConstDrop` with precise drop checking. This is the only const
- // check performed after the promotion. Verify that with an assertion.
- assert!(promoted.is_none() || Q::ALLOW_PROMOTED);
- // Don't peek inside trait associated constants.
- if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() {
- let qualifs = if let Some((did, param_did)) = def.as_const_arg() {
- cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did))
- } else {
- cx.tcx.at(constant.span).mir_const_qualif(def.did)
- };
-
- if !Q::in_qualifs(&qualifs) {
- return false;
- }
+ // FIXME(valtrees): check whether const qualifs should behave the same
+ // way for type and mir constants.
+ let uneval = match constant.literal {
+ ConstantKind::Ty(ct) if matches!(ct.kind(), ty::ConstKind::Unevaluated(_)) => {
+ let ty::ConstKind::Unevaluated(uv) = ct.kind() else { unreachable!() };
+
+ Some(uv.expand())
+ }
+ ConstantKind::Ty(_) => None,
+ ConstantKind::Unevaluated(uv, _) => Some(uv),
+ ConstantKind::Val(..) => None,
+ };
- // Just in case the type is more specific than
- // the definition, e.g., impl associated const
- // with type parameters, take it into account.
+ if let Some(ty::Unevaluated { def, substs: _, promoted }) = uneval {
+ // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible
+ // only for `NeedsNonConstDrop` with precise drop checking. This is the only const
+ // check performed after the promotion. Verify that with an assertion.
+ assert!(promoted.is_none() || Q::ALLOW_PROMOTED);
+
+ // Don't peek inside trait associated constants.
+ if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() {
+ let qualifs = if let Some((did, param_did)) = def.as_const_arg() {
+ cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did))
+ } else {
+ cx.tcx.at(constant.span).mir_const_qualif(def.did)
+ };
+
+ if !Q::in_qualifs(&qualifs) {
+ return false;
}
+
+ // Just in case the type is more specific than
+ // the definition, e.g., impl associated const
+ // with type parameters, take it into account.
}
}
+
// Otherwise use the qualifs of the type.
Q::in_any_value_of_ty(cx, constant.literal.ty())
}
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index ed4d8c95d..f7a7cc88a 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -41,7 +41,7 @@ pub struct PromoteTemps<'tcx> {
impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
fn phase_change(&self) -> Option<MirPhase> {
- Some(MirPhase::ConstsPromoted)
+ Some(MirPhase::Analysis(AnalysisPhase::Initial))
}
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -710,7 +710,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
}
fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) {
- let last = self.promoted.basic_blocks().last().unwrap();
+ let last = self.promoted.basic_blocks.last().unwrap();
let data = &mut self.promoted[last];
data.statements.push(Statement {
source_info: SourceInfo::outermost(span),
@@ -803,7 +803,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
self.visit_operand(arg, loc);
}
- let last = self.promoted.basic_blocks().last().unwrap();
+ let last = self.promoted.basic_blocks.last().unwrap();
let new_target = self.new_block();
*self.promoted[last].terminator_mut() = Terminator {
@@ -839,27 +839,16 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
let mut promoted_operand = |ty, span| {
promoted.span = span;
promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span);
- let _const = tcx.mk_const(ty::ConstS {
- ty,
- kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
- def,
- substs: InternalSubsts::for_item(tcx, def.did, |param, _| {
- if let ty::GenericParamDefKind::Lifetime = param.kind {
- tcx.lifetimes.re_erased.into()
- } else {
- tcx.mk_param_from_def(param)
- }
- }),
- promoted: Some(promoted_id),
- }),
- });
+ let substs = tcx.erase_regions(InternalSubsts::identity_for_item(tcx, def.did));
+ let uneval = ty::Unevaluated { def, substs, promoted: Some(promoted_id) };
Operand::Constant(Box::new(Constant {
span,
user_ty: None,
- literal: ConstantKind::from_const(_const, tcx),
+ literal: ConstantKind::Unevaluated(uneval, ty),
}))
};
+
let blocks = self.source.basic_blocks.as_mut();
let local_decls = &mut self.source.local_decls;
let loc = candidate.location;
@@ -969,7 +958,7 @@ pub fn promote_candidates<'tcx>(
let mut scope = body.source_scopes[body.source_info(candidate.location).scope].clone();
scope.parent_scope = None;
- let promoted = Body::new(
+ let mut promoted = Body::new(
body.source, // `promoted` gets filled in below
IndexVec::new(),
IndexVec::from_elem_n(scope, 1),
@@ -981,6 +970,7 @@ pub fn promote_candidates<'tcx>(
body.generator_kind(),
body.tainted_by_errors,
);
+ promoted.phase = MirPhase::Analysis(AnalysisPhase::Initial);
let promoter = Promoter {
promoted,
@@ -1046,7 +1036,7 @@ pub fn is_const_fn_in_array_repeat_expression<'tcx>(
_ => {}
}
- for block in body.basic_blocks() {
+ for block in body.basic_blocks.iter() {
if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) =
&block.terminator
{
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index 15e820f2d..4aa98cb13 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar;
use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
use rustc_middle::mir::visit::{PlaceContext, Visitor};
use rustc_middle::mir::{
- traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location,
- MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, Rvalue, SourceScope,
- Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK,
+ traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
+ Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
+ ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
+ TerminatorKind, UnOp, START_BLOCK,
};
use rustc_middle::ty::fold::BottomUpFolder;
use rustc_middle::ty::subst::Subst;
@@ -89,9 +90,8 @@ pub fn equal_up_to_regions<'tcx>(
// Normalize lifetimes away on both sides, then compare.
let normalize = |ty: Ty<'tcx>| {
- tcx.normalize_erasing_regions(
- param_env,
- ty.fold_with(&mut BottomUpFolder {
+ tcx.try_normalize_erasing_regions(param_env, ty).unwrap_or(ty).fold_with(
+ &mut BottomUpFolder {
tcx,
// FIXME: We erase all late-bound lifetimes, but this is not fully correct.
// If you have a type like `<for<'a> fn(&'a u32) as SomeTrait>::Assoc`,
@@ -103,7 +103,7 @@ pub fn equal_up_to_regions<'tcx>(
// Leave consts and types unchanged.
ct_op: |ct| ct,
ty_op: |ty| ty,
- }),
+ },
)
};
tcx.infer_ctxt().enter(|infcx| infcx.can_eq(param_env, normalize(src), normalize(dest)).is_ok())
@@ -142,8 +142,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
if bb == START_BLOCK {
self.fail(location, "start block must not have predecessors")
}
- if let Some(bb) = self.body.basic_blocks().get(bb) {
- let src = self.body.basic_blocks().get(location.block).unwrap();
+ if let Some(bb) = self.body.basic_blocks.get(bb) {
+ let src = self.body.basic_blocks.get(location.block).unwrap();
match (src.is_cleanup, bb.is_cleanup, edge_kind) {
// Non-cleanup blocks can jump to non-cleanup blocks along non-unwind edges
(false, false, EdgeKind::Normal)
@@ -183,16 +183,23 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
if (src, dest).has_opaque_types() {
return true;
}
- // Normalize projections and things like that.
- let param_env = self.param_env.with_reveal_all_normalized(self.tcx);
- let src = self.tcx.normalize_erasing_regions(param_env, src);
- let dest = self.tcx.normalize_erasing_regions(param_env, dest);
+ // Normalize projections and things like that.
// Type-changing assignments can happen when subtyping is used. While
// all normal lifetimes are erased, higher-ranked types with their
// late-bound lifetimes are still around and can lead to type
// differences. So we compare ignoring lifetimes.
- equal_up_to_regions(self.tcx, param_env, src, dest)
+
+ // First, try with reveal_all. This might not work in some cases, as the predicates
+ // can be cleared in reveal_all mode. We try the reveal first anyways as it is used
+ // by some other passes like inlining as well.
+ let param_env = self.param_env.with_reveal_all_normalized(self.tcx);
+ if equal_up_to_regions(self.tcx, param_env, src, dest) {
+ return true;
+ }
+
+ // If this fails, we can try it without the reveal.
+ equal_up_to_regions(self.tcx, self.param_env, src, dest)
}
}
@@ -223,7 +230,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
// This check is somewhat expensive, so only run it when -Zvalidate-mir is passed.
- if self.tcx.sess.opts.unstable_opts.validate_mir && self.mir_phase < MirPhase::DropsLowered
+ if self.tcx.sess.opts.unstable_opts.validate_mir
+ && self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial)
{
// `Operand::Copy` is only supposed to be used with `Copy` types.
if let Operand::Copy(place) = operand {
@@ -254,7 +262,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
self.fail(location, format!("bad index ({:?} != usize)", index_ty))
}
}
- ProjectionElem::Deref if self.mir_phase >= MirPhase::GeneratorsLowered => {
+ ProjectionElem::Deref
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) =>
+ {
let base_ty = Place::ty_from(local, proj_base, &self.body.local_decls, self.tcx).ty;
if base_ty.is_box() {
@@ -362,7 +372,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
// Set off any `bug!`s in the type computation code
let _ = place.ty(&self.body.local_decls, self.tcx);
- if self.mir_phase >= MirPhase::Derefered
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial)
&& place.projection.len() > 1
&& cntxt != PlaceContext::NonUse(VarDebugInfo)
&& place.projection[1..].contains(&ProjectionElem::Deref)
@@ -386,8 +396,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
Rvalue::Aggregate(agg_kind, _) => {
let disallowed = match **agg_kind {
AggregateKind::Array(..) => false,
- AggregateKind::Generator(..) => self.mir_phase >= MirPhase::GeneratorsLowered,
- _ => self.mir_phase >= MirPhase::Deaggregated,
+ _ => self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup),
};
if disallowed {
self.fail(
@@ -397,10 +406,10 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
Rvalue::Ref(_, BorrowKind::Shallow, _) => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
- "`Assign` statement with a `Shallow` borrow should have been removed after drop lowering phase",
+ "`Assign` statement with a `Shallow` borrow should have been removed in runtime MIR",
);
}
}
@@ -560,6 +569,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
);
}
}
+ CastKind::DynStar => {
+ // FIXME(dyn-star): make sure nothing needs to be done here.
+ }
// Nothing to check here
CastKind::PointerFromExposedAddress
| CastKind::PointerExposeAddress
@@ -614,7 +626,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
StatementKind::AscribeUserType(..) => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`AscribeUserType` should have been removed after drop lowering phase",
@@ -622,18 +634,25 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
StatementKind::FakeRead(..) => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`FakeRead` should have been removed after drop lowering phase",
);
}
}
- StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
- ref src,
- ref dst,
- ref count,
- }) => {
+ StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
+ let ty = op.ty(&self.body.local_decls, self.tcx);
+ if !ty.is_bool() {
+ self.fail(
+ location,
+ format!("`assume` argument must be `bool`, but got: `{}`", ty),
+ );
+ }
+ }
+ StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+ CopyNonOverlapping { src, dst, count },
+ )) => {
let src_ty = src.ty(&self.body.local_decls, self.tcx);
let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
src_deref.ty
@@ -666,7 +685,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
StatementKind::SetDiscriminant { place, .. } => {
- if self.mir_phase < MirPhase::Deaggregated {
+ if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(location, "`SetDiscriminant`is not allowed until deaggregation");
}
let pty = place.ty(&self.body.local_decls, self.tcx).ty.kind();
@@ -681,7 +700,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
StatementKind::Deinit(..) => {
- if self.mir_phase < MirPhase::Deaggregated {
+ if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(location, "`Deinit`is not allowed until deaggregation");
}
}
@@ -761,7 +780,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
TerminatorKind::DropAndReplace { target, unwind, .. } => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`DropAndReplace` should have been removed during drop elaboration",
@@ -832,7 +851,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
if self.body.generator.is_none() {
self.fail(location, "`Yield` cannot appear outside generator bodies");
}
- if self.mir_phase >= MirPhase::GeneratorsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(location, "`Yield` should have been replaced by generator lowering");
}
self.check_edge(location, *resume, EdgeKind::Normal);
@@ -841,7 +860,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
}
TerminatorKind::FalseEdge { real_target, imaginary_target } => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`FalseEdge` should have been removed after drop elaboration",
@@ -851,7 +870,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
self.check_edge(location, *imaginary_target, EdgeKind::Normal);
}
TerminatorKind::FalseUnwind { real_target, unwind } => {
- if self.mir_phase >= MirPhase::DropsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`FalseUnwind` should have been removed after drop elaboration",
@@ -874,7 +893,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
if self.body.generator.is_none() {
self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies");
}
- if self.mir_phase >= MirPhase::GeneratorsLowered {
+ if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
self.fail(
location,
"`GeneratorDrop` should have been replaced by generator lowering",
@@ -883,13 +902,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
}
TerminatorKind::Resume | TerminatorKind::Abort => {
let bb = location.block;
- if !self.body.basic_blocks()[bb].is_cleanup {
+ if !self.body.basic_blocks[bb].is_cleanup {
self.fail(location, "Cannot `Resume` or `Abort` from non-cleanup basic block")
}
}
TerminatorKind::Return => {
let bb = location.block;
- if self.body.basic_blocks()[bb].is_cleanup {
+ if self.body.basic_blocks[bb].is_cleanup {
self.fail(location, "Cannot `Return` from cleanup basic block")
}
}