diff options
Diffstat (limited to 'compiler/rustc_middle/src/mir/interpret')
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/allocation.rs | 331 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/error.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/mod.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/pointer.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/queries.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/value.rs | 163 |
6 files changed, 175 insertions, 357 deletions
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs index db7e0fb8a..37ec04b07 100644 --- a/compiler/rustc_middle/src/mir/interpret/allocation.rs +++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs @@ -16,8 +16,8 @@ use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer, Provenance, - ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, ScalarSizeMismatch, UndefinedBehaviorInfo, - UninitBytesAccess, UnsupportedOpInfo, + ResourceExhaustionInfo, Scalar, ScalarSizeMismatch, UndefinedBehaviorInfo, UninitBytesAccess, + UnsupportedOpInfo, }; use crate::ty; @@ -34,11 +34,11 @@ pub struct Allocation<Prov = AllocId, Extra = ()> { /// The actual bytes of the allocation. /// Note that the bytes of a pointer represent the offset of the pointer. bytes: Box<[u8]>, - /// Maps from byte addresses to extra data for each pointer. + /// Maps from byte addresses to extra provenance data for each pointer. /// Only the first byte of a pointer is inserted into the map; i.e., /// every entry in this map applies to `pointer_size` consecutive bytes starting /// at the given offset. - relocations: Relocations<Prov>, + provenance: ProvenanceMap<Prov>, /// Denotes which part of this allocation is initialized. init_mask: InitMask, /// The alignment of the allocation to detect unaligned reads. @@ -84,7 +84,7 @@ impl hash::Hash for Allocation { } // Hash the other fields as usual. - self.relocations.hash(state); + self.provenance.hash(state); self.init_mask.hash(state); self.align.hash(state); self.mutability.hash(state); @@ -130,6 +130,8 @@ pub enum AllocError { ReadPointerAsBytes, /// Partially overwriting a pointer. PartialPointerOverwrite(Size), + /// Partially copying a pointer. + PartialPointerCopy(Size), /// Using uninitialized data where it is not allowed. InvalidUninitBytes(Option<UninitBytesAccess>), } @@ -152,6 +154,9 @@ impl AllocError { PartialPointerOverwrite(offset) => InterpError::Unsupported( UnsupportedOpInfo::PartialPointerOverwrite(Pointer::new(alloc_id, offset)), ), + PartialPointerCopy(offset) => InterpError::Unsupported( + UnsupportedOpInfo::PartialPointerCopy(Pointer::new(alloc_id, offset)), + ), InvalidUninitBytes(info) => InterpError::UndefinedBehavior( UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))), ), @@ -211,7 +216,7 @@ impl<Prov> Allocation<Prov> { let size = Size::from_bytes(bytes.len()); Self { bytes, - relocations: Relocations::new(), + provenance: ProvenanceMap::new(), init_mask: InitMask::new(size, true), align, mutability, @@ -246,7 +251,7 @@ impl<Prov> Allocation<Prov> { let bytes = unsafe { bytes.assume_init() }; Ok(Allocation { bytes, - relocations: Relocations::new(), + provenance: ProvenanceMap::new(), init_mask: InitMask::new(size, false), align, mutability: Mutability::Mut, @@ -266,22 +271,22 @@ impl Allocation { ) -> Result<Allocation<Prov, Extra>, Err> { // Compute new pointer provenance, which also adjusts the bytes. let mut bytes = self.bytes; - let mut new_relocations = Vec::with_capacity(self.relocations.0.len()); + let mut new_provenance = Vec::with_capacity(self.provenance.0.len()); let ptr_size = cx.data_layout().pointer_size.bytes_usize(); let endian = cx.data_layout().endian; - for &(offset, alloc_id) in self.relocations.iter() { + for &(offset, alloc_id) in self.provenance.iter() { let idx = offset.bytes_usize(); let ptr_bytes = &mut bytes[idx..idx + ptr_size]; let bits = read_target_uint(endian, ptr_bytes).unwrap(); let (ptr_prov, ptr_offset) = adjust_ptr(Pointer::new(alloc_id, Size::from_bytes(bits)))?.into_parts(); write_target_uint(endian, ptr_bytes, ptr_offset.bytes().into()).unwrap(); - new_relocations.push((offset, ptr_prov)); + new_provenance.push((offset, ptr_prov)); } // Create allocation. Ok(Allocation { bytes, - relocations: Relocations::from_presorted(new_relocations), + provenance: ProvenanceMap::from_presorted(new_provenance), init_mask: self.init_mask, align: self.align, mutability: self.mutability, @@ -300,8 +305,8 @@ impl<Prov, Extra> Allocation<Prov, Extra> { Size::from_bytes(self.len()) } - /// Looks at a slice which may describe uninitialized bytes or describe a relocation. This differs - /// from `get_bytes_with_uninit_and_ptr` in that it does no relocation checks (even on the + /// Looks at a slice which may contain uninitialized bytes or provenance. This differs + /// from `get_bytes_with_uninit_and_ptr` in that it does no provenance checks (even on the /// edges) at all. /// This must not be used for reads affecting the interpreter execution. pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range<usize>) -> &[u8] { @@ -313,74 +318,47 @@ impl<Prov, Extra> Allocation<Prov, Extra> { &self.init_mask } - /// Returns the relocation list. - pub fn relocations(&self) -> &Relocations<Prov> { - &self.relocations + /// Returns the provenance map. + pub fn provenance(&self) -> &ProvenanceMap<Prov> { + &self.provenance } } /// Byte accessors. impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { /// This is the entirely abstraction-violating way to just grab the raw bytes without - /// caring about relocations. It just deduplicates some code between `read_scalar` - /// and `get_bytes_internal`. - fn get_bytes_even_more_internal(&self, range: AllocRange) -> &[u8] { - &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()] - } - - /// The last argument controls whether we error out when there are uninitialized or pointer - /// bytes. However, we *always* error when there are relocations overlapping the edges of the - /// range. - /// - /// You should never call this, call `get_bytes` or `get_bytes_with_uninit_and_ptr` instead, + /// caring about provenance or initialization. /// /// This function also guarantees that the resulting pointer will remain stable /// even when new allocations are pushed to the `HashMap`. `mem_copy_repeatedly` relies /// on that. - /// - /// It is the caller's responsibility to check bounds and alignment beforehand. - fn get_bytes_internal( - &self, - cx: &impl HasDataLayout, - range: AllocRange, - check_init_and_ptr: bool, - ) -> AllocResult<&[u8]> { - if check_init_and_ptr { - self.check_init(range)?; - self.check_relocations(cx, range)?; - } else { - // We still don't want relocations on the *edges*. - self.check_relocation_edges(cx, range)?; - } - - Ok(self.get_bytes_even_more_internal(range)) + #[inline] + pub fn get_bytes_unchecked(&self, range: AllocRange) -> &[u8] { + &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()] } - /// Checks that these bytes are initialized and not pointer bytes, and then return them - /// as a slice. + /// Checks that these bytes are initialized, and then strip provenance (if possible) and return + /// them. /// /// It is the caller's responsibility to check bounds and alignment beforehand. /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods /// on `InterpCx` instead. #[inline] - pub fn get_bytes(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult<&[u8]> { - self.get_bytes_internal(cx, range, true) - } - - /// It is the caller's responsibility to handle uninitialized and pointer bytes. - /// However, this still checks that there are no relocations on the *edges*. - /// - /// It is the caller's responsibility to check bounds and alignment beforehand. - #[inline] - pub fn get_bytes_with_uninit_and_ptr( + pub fn get_bytes_strip_provenance( &self, cx: &impl HasDataLayout, range: AllocRange, ) -> AllocResult<&[u8]> { - self.get_bytes_internal(cx, range, false) + self.check_init(range)?; + if !Prov::OFFSET_IS_ADDR { + if self.range_has_provenance(cx, range) { + return Err(AllocError::ReadPointerAsBytes); + } + } + Ok(self.get_bytes_unchecked(range)) } - /// Just calling this already marks everything as defined and removes relocations, + /// Just calling this already marks everything as defined and removes provenance, /// so be sure to actually put data there! /// /// It is the caller's responsibility to check bounds and alignment beforehand. @@ -392,7 +370,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { range: AllocRange, ) -> AllocResult<&mut [u8]> { self.mark_init(range, true); - self.clear_relocations(cx, range)?; + self.clear_provenance(cx, range)?; Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]) } @@ -404,7 +382,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { range: AllocRange, ) -> AllocResult<*mut [u8]> { self.mark_init(range, true); - self.clear_relocations(cx, range)?; + self.clear_provenance(cx, range)?; assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize()); @@ -415,28 +393,6 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { /// Reading and writing. impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { - /// Validates that `ptr.offset` and `ptr.offset + size` do not point to the middle of a - /// relocation. If `allow_uninit`/`allow_ptr` is `false`, also enforces that the memory in the - /// given range contains no uninitialized bytes/relocations. - pub fn check_bytes( - &self, - cx: &impl HasDataLayout, - range: AllocRange, - allow_uninit: bool, - allow_ptr: bool, - ) -> AllocResult { - // Check bounds and relocations on the edges. - self.get_bytes_with_uninit_and_ptr(cx, range)?; - // Check uninit and ptr. - if !allow_uninit { - self.check_init(range)?; - } - if !allow_ptr { - self.check_relocations(cx, range)?; - } - Ok(()) - } - /// Reads a *non-ZST* scalar. /// /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine @@ -452,47 +408,55 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { cx: &impl HasDataLayout, range: AllocRange, read_provenance: bool, - ) -> AllocResult<ScalarMaybeUninit<Prov>> { - if read_provenance { - assert_eq!(range.size, cx.data_layout().pointer_size); - } - + ) -> AllocResult<Scalar<Prov>> { // First and foremost, if anything is uninit, bail. if self.is_init(range).is_err() { - // This inflates uninitialized bytes to the entire scalar, even if only a few - // bytes are uninitialized. - return Ok(ScalarMaybeUninit::Uninit); + return Err(AllocError::InvalidUninitBytes(None)); } - // If we are doing a pointer read, and there is a relocation exactly where we - // are reading, then we can put data and relocation back together and return that. - if read_provenance && let Some(&prov) = self.relocations.get(&range.start) { - // We already checked init and relocations, so we can use this function. - let bytes = self.get_bytes_even_more_internal(range); - let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap(); - let ptr = Pointer::new(prov, Size::from_bytes(bits)); - return Ok(ScalarMaybeUninit::from_pointer(ptr, cx)); - } + // Get the integer part of the result. We HAVE TO check provenance before returning this! + let bytes = self.get_bytes_unchecked(range); + let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap(); - // If we are *not* reading a pointer, and we can just ignore relocations, - // then do exactly that. - if !read_provenance && Prov::OFFSET_IS_ADDR { - // We just strip provenance. - let bytes = self.get_bytes_even_more_internal(range); - let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap(); - return Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size))); + if read_provenance { + assert_eq!(range.size, cx.data_layout().pointer_size); + + // When reading data with provenance, the easy case is finding provenance exactly where we + // are reading, then we can put data and provenance back together and return that. + if let Some(&prov) = self.provenance.get(&range.start) { + // Now we can return the bits, with their appropriate provenance. + let ptr = Pointer::new(prov, Size::from_bytes(bits)); + return Ok(Scalar::from_pointer(ptr, cx)); + } + + // If we can work on pointers byte-wise, join the byte-wise provenances. + if Prov::OFFSET_IS_ADDR { + let mut prov = self.offset_get_provenance(cx, range.start); + for offset in 1..range.size.bytes() { + let this_prov = + self.offset_get_provenance(cx, range.start + Size::from_bytes(offset)); + prov = Prov::join(prov, this_prov); + } + // Now use this provenance. + let ptr = Pointer::new(prov, Size::from_bytes(bits)); + return Ok(Scalar::from_maybe_pointer(ptr, cx)); + } + } else { + // We are *not* reading a pointer. + // If we can just ignore provenance, do exactly that. + if Prov::OFFSET_IS_ADDR { + // We just strip provenance. + return Ok(Scalar::from_uint(bits, range.size)); + } } - // It's complicated. Better make sure there is no provenance anywhere. - // FIXME: If !OFFSET_IS_ADDR, this is the best we can do. But if OFFSET_IS_ADDR, then - // `read_pointer` is true and we ideally would distinguish the following two cases: - // - The entire `range` is covered by 2 relocations for the same provenance. - // Then we should return a pointer with that provenance. - // - The range has inhomogeneous provenance. Then we should return just the - // underlying bits. - let bytes = self.get_bytes(cx, range)?; - let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap(); - Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size))) + // Fallback path for when we cannot treat provenance bytewise or ignore it. + assert!(!Prov::OFFSET_IS_ADDR); + if self.range_has_provenance(cx, range) { + return Err(AllocError::ReadPointerAsBytes); + } + // There is no provenance, we can just return the bits. + Ok(Scalar::from_uint(bits, range.size)) } /// Writes a *non-ZST* scalar. @@ -507,17 +471,10 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { &mut self, cx: &impl HasDataLayout, range: AllocRange, - val: ScalarMaybeUninit<Prov>, + val: Scalar<Prov>, ) -> AllocResult { assert!(self.mutability == Mutability::Mut); - let val = match val { - ScalarMaybeUninit::Scalar(scalar) => scalar, - ScalarMaybeUninit::Uninit => { - return self.write_uninit(cx, range); - } - }; - // `to_bits_or_ptr_internal` is the right method because we just want to store this data // as-is into memory. let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? { @@ -532,9 +489,9 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { let dst = self.get_bytes_mut(cx, range)?; write_target_uint(endian, dst, bytes).unwrap(); - // See if we have to also write a relocation. + // See if we have to also store some provenance. if let Some(provenance) = provenance { - self.relocations.0.insert(range.start, provenance); + self.provenance.0.insert(range.start, provenance); } Ok(()) @@ -543,64 +500,65 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> { /// Write "uninit" to the given memory range. pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { self.mark_init(range, false); - self.clear_relocations(cx, range)?; + self.clear_provenance(cx, range)?; return Ok(()); } } -/// Relocations. +/// Provenance. impl<Prov: Copy, Extra> Allocation<Prov, Extra> { - /// Returns all relocations overlapping with the given pointer-offset pair. - fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Prov)] { + /// Returns all provenance overlapping with the given pointer-offset pair. + fn range_get_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Prov)] { // We have to go back `pointer_size - 1` bytes, as that one would still overlap with // the beginning of this range. let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1); - self.relocations.range(Size::from_bytes(start)..range.end()) + self.provenance.range(Size::from_bytes(start)..range.end()) } - /// Returns whether this allocation has relocations overlapping with the given range. - /// - /// Note: this function exists to allow `get_relocations` to be private, in order to somewhat - /// limit access to relocations outside of the `Allocation` abstraction. - /// - pub fn has_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool { - !self.get_relocations(cx, range).is_empty() + /// Get the provenance of a single byte. + fn offset_get_provenance(&self, cx: &impl HasDataLayout, offset: Size) -> Option<Prov> { + let prov = self.range_get_provenance(cx, alloc_range(offset, Size::from_bytes(1))); + assert!(prov.len() <= 1); + prov.first().map(|(_offset, prov)| *prov) } - /// Checks that there are no relocations overlapping with the given range. - #[inline(always)] - fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { - if self.has_relocations(cx, range) { Err(AllocError::ReadPointerAsBytes) } else { Ok(()) } + /// Returns whether this allocation has progrnance overlapping with the given range. + /// + /// Note: this function exists to allow `range_get_provenance` to be private, in order to somewhat + /// limit access to provenance outside of the `Allocation` abstraction. + /// + pub fn range_has_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool { + !self.range_get_provenance(cx, range).is_empty() } - /// Removes all relocations inside the given range. - /// If there are relocations overlapping with the edges, they + /// Removes all provenance inside the given range. + /// If there is provenance overlapping with the edges, it /// are removed as well *and* the bytes they cover are marked as /// uninitialized. This is a somewhat odd "spooky action at a distance", /// but it allows strictly more code to run than if we would just error /// immediately in that case. - fn clear_relocations(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult + fn clear_provenance(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult where Prov: Provenance, { - // Find the start and end of the given range and its outermost relocations. + // Find the start and end of the given range and its outermost provenance. let (first, last) = { - // Find all relocations overlapping the given range. - let relocations = self.get_relocations(cx, range); - if relocations.is_empty() { + // Find all provenance overlapping the given range. + let provenance = self.range_get_provenance(cx, range); + if provenance.is_empty() { return Ok(()); } ( - relocations.first().unwrap().0, - relocations.last().unwrap().0 + cx.data_layout().pointer_size, + provenance.first().unwrap().0, + provenance.last().unwrap().0 + cx.data_layout().pointer_size, ) }; let start = range.start; let end = range.end(); - // We need to handle clearing the relocations from parts of a pointer. - // FIXME: Miri should preserve partial relocations; see + // We need to handle clearing the provenance from parts of a pointer. + // FIXME: Miri should preserve partial provenance; see // https://github.com/rust-lang/miri/issues/2181. if first < start { if Prov::ERR_ON_PARTIAL_PTR_OVERWRITE { @@ -623,41 +581,32 @@ impl<Prov: Copy, Extra> Allocation<Prov, Extra> { self.init_mask.set_range(end, last, false); } - // Forget all the relocations. - // Since relocations do not overlap, we know that removing until `last` (exclusive) is fine, - // i.e., this will not remove any other relocations just after the ones we care about. - self.relocations.0.remove_range(first..last); + // Forget all the provenance. + // Since provenance do not overlap, we know that removing until `last` (exclusive) is fine, + // i.e., this will not remove any other provenance just after the ones we care about. + self.provenance.0.remove_range(first..last); Ok(()) } - - /// Errors if there are relocations overlapping with the edges of the - /// given memory range. - #[inline] - fn check_relocation_edges(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { - self.check_relocations(cx, alloc_range(range.start, Size::ZERO))?; - self.check_relocations(cx, alloc_range(range.end(), Size::ZERO))?; - Ok(()) - } } -/// "Relocations" stores the provenance information of pointers stored in memory. +/// Stores the provenance information of pointers stored in memory. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)] -pub struct Relocations<Prov = AllocId>(SortedMap<Size, Prov>); +pub struct ProvenanceMap<Prov = AllocId>(SortedMap<Size, Prov>); -impl<Prov> Relocations<Prov> { +impl<Prov> ProvenanceMap<Prov> { pub fn new() -> Self { - Relocations(SortedMap::new()) + ProvenanceMap(SortedMap::new()) } - // The caller must guarantee that the given relocations are already sorted + // The caller must guarantee that the given provenance list is already sorted // by address and contain no duplicates. pub fn from_presorted(r: Vec<(Size, Prov)>) -> Self { - Relocations(SortedMap::from_presorted_elements(r)) + ProvenanceMap(SortedMap::from_presorted_elements(r)) } } -impl<Prov> Deref for Relocations<Prov> { +impl<Prov> Deref for ProvenanceMap<Prov> { type Target = SortedMap<Size, Prov>; fn deref(&self) -> &Self::Target { @@ -665,36 +614,36 @@ impl<Prov> Deref for Relocations<Prov> { } } -/// A partial, owned list of relocations to transfer into another allocation. +/// A partial, owned list of provenance to transfer into another allocation. /// /// Offsets are already adjusted to the destination allocation. -pub struct AllocationRelocations<Prov> { - dest_relocations: Vec<(Size, Prov)>, +pub struct AllocationProvenance<Prov> { + dest_provenance: Vec<(Size, Prov)>, } impl<Prov: Copy, Extra> Allocation<Prov, Extra> { - pub fn prepare_relocation_copy( + pub fn prepare_provenance_copy( &self, cx: &impl HasDataLayout, src: AllocRange, dest: Size, count: u64, - ) -> AllocationRelocations<Prov> { - let relocations = self.get_relocations(cx, src); - if relocations.is_empty() { - return AllocationRelocations { dest_relocations: Vec::new() }; + ) -> AllocationProvenance<Prov> { + let provenance = self.range_get_provenance(cx, src); + if provenance.is_empty() { + return AllocationProvenance { dest_provenance: Vec::new() }; } let size = src.size; - let mut new_relocations = Vec::with_capacity(relocations.len() * (count as usize)); + let mut new_provenance = Vec::with_capacity(provenance.len() * (count as usize)); // If `count` is large, this is rather wasteful -- we are allocating a big array here, which // is mostly filled with redundant information since it's just N copies of the same `Prov`s - // at slightly adjusted offsets. The reason we do this is so that in `mark_relocation_range` + // at slightly adjusted offsets. The reason we do this is so that in `mark_provenance_range` // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces - // the right sequence of relocations for all N copies. + // the right sequence of provenance for all N copies. for i in 0..count { - new_relocations.extend(relocations.iter().map(|&(offset, reloc)| { + new_provenance.extend(provenance.iter().map(|&(offset, reloc)| { // compute offset for current repetition let dest_offset = dest + size * i; // `Size` operations ( @@ -705,17 +654,17 @@ impl<Prov: Copy, Extra> Allocation<Prov, Extra> { })); } - AllocationRelocations { dest_relocations: new_relocations } + AllocationProvenance { dest_provenance: new_provenance } } - /// Applies a relocation copy. - /// The affected range, as defined in the parameters to `prepare_relocation_copy` is expected - /// to be clear of relocations. + /// Applies a provenance copy. + /// The affected range, as defined in the parameters to `prepare_provenance_copy` is expected + /// to be clear of provenance. /// /// This is dangerous to use as it can violate internal `Allocation` invariants! /// It only exists to support an efficient implementation of `mem_copy_repeatedly`. - pub fn mark_relocation_range(&mut self, relocations: AllocationRelocations<Prov>) { - self.relocations.0.insert_presorted(relocations.dest_relocations); + pub fn mark_provenance_range(&mut self, provenance: AllocationProvenance<Prov>) { + self.provenance.0.insert_presorted(provenance.dest_provenance); } } diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index cecb55578..e4039cc7c 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -401,14 +401,18 @@ impl fmt::Display for UndefinedBehaviorInfo { pub enum UnsupportedOpInfo { /// Free-form case. Only for errors that are never caught! Unsupported(String), - /// Encountered a pointer where we needed raw bytes. - ReadPointerAsBytes, /// Overwriting parts of a pointer; the resulting state cannot be represented in our /// `Allocation` data structure. See <https://github.com/rust-lang/miri/issues/2181>. PartialPointerOverwrite(Pointer<AllocId>), + /// Attempting to `copy` parts of a pointer to somewhere else; the resulting state cannot be + /// represented in our `Allocation` data structure. See + /// <https://github.com/rust-lang/miri/issues/2181>. + PartialPointerCopy(Pointer<AllocId>), // // The variants below are only reachable from CTFE/const prop, miri will never emit them. // + /// Encountered a pointer where we needed raw bytes. + ReadPointerAsBytes, /// Accessing thread local statics ThreadLocalStatic(DefId), /// Accessing an unsupported extern static. @@ -420,10 +424,13 @@ impl fmt::Display for UnsupportedOpInfo { use UnsupportedOpInfo::*; match self { Unsupported(ref msg) => write!(f, "{msg}"), - ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes"), PartialPointerOverwrite(ptr) => { write!(f, "unable to overwrite parts of a pointer in memory at {ptr:?}") } + PartialPointerCopy(ptr) => { + write!(f, "unable to copy parts of a pointer from memory at {ptr:?}") + } + ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes"), ThreadLocalStatic(did) => write!(f, "cannot access thread local static ({did:?})"), ReadExternStatic(did) => write!(f, "cannot read from extern static ({did:?})"), } diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs index 967f8ece1..5e3dfcbcc 100644 --- a/compiler/rustc_middle/src/mir/interpret/mod.rs +++ b/compiler/rustc_middle/src/mir/interpret/mod.rs @@ -124,11 +124,11 @@ pub use self::error::{ UninitBytesAccess, UnsupportedOpInfo, }; -pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit}; +pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar}; pub use self::allocation::{ alloc_range, AllocRange, Allocation, ConstAllocation, InitChunk, InitChunkIter, InitMask, - Relocations, + ProvenanceMap, }; pub use self::pointer::{Pointer, PointerArithmetic, Provenance}; @@ -137,7 +137,7 @@ pub use self::pointer::{Pointer, PointerArithmetic, Provenance}; /// - A constant /// - A static #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, TyEncodable, TyDecodable)] -#[derive(HashStable, Lift)] +#[derive(HashStable, Lift, TypeFoldable, TypeVisitable)] pub struct GlobalId<'tcx> { /// For a constant or static, the `Instance` of the item itself. /// For a promoted global, the `Instance` of the function they belong to. diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs index 384954cbb..95e52e391 100644 --- a/compiler/rustc_middle/src/mir/interpret/pointer.rs +++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs @@ -107,8 +107,12 @@ impl<T: HasDataLayout> PointerArithmetic for T {} /// pointer), but `derive` adds some unnecessary bounds. pub trait Provenance: Copy + fmt::Debug { /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address. - /// If `true, ptr-to-int casts work by simply discarding the provenance. - /// If `false`, ptr-to-int casts are not supported. The offset *must* be relative in that case. + /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are + /// different from what the Abstract Machine prescribes, so the interpreter must prevent any + /// operation that would inspect the underlying bytes of a pointer, such as ptr-to-int + /// transmutation. A `ReadPointerAsBytes` error will be raised in such situations. + /// - If `true`, the interpreter will permit operations to inspect the underlying bytes of a + /// pointer, and implement ptr-to-int transmutation by stripping provenance. const OFFSET_IS_ADDR: bool; /// We also use this trait to control whether to abort execution when a pointer is being partially overwritten @@ -125,6 +129,9 @@ pub trait Provenance: Copy + fmt::Debug { /// Otherwise this function is best-effort (but must agree with `Machine::ptr_get_alloc`). /// (Identifying the offset in that allocation, however, is harder -- use `Memory::ptr_get_alloc` for that.) fn get_alloc_id(self) -> Option<AllocId>; + + /// Defines the 'join' of provenance: what happens when doing a pointer load and different bytes have different provenance. + fn join(left: Option<Self>, right: Option<Self>) -> Option<Self>; } impl Provenance for AllocId { @@ -152,6 +159,10 @@ impl Provenance for AllocId { fn get_alloc_id(self) -> Option<AllocId> { Some(self) } + + fn join(_left: Option<Self>, _right: Option<Self>) -> Option<Self> { + panic!("merging provenance is not supported when `OFFSET_IS_ADDR` is false") + } } /// Represents a pointer in the Miri engine. diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs index 786927e2d..4207988d7 100644 --- a/compiler/rustc_middle/src/mir/interpret/queries.rs +++ b/compiler/rustc_middle/src/mir/interpret/queries.rs @@ -63,7 +63,7 @@ impl<'tcx> TyCtxt<'tcx> { pub fn const_eval_resolve_for_typeck( self, param_env: ty::ParamEnv<'tcx>, - ct: ty::Unevaluated<'tcx>, + ct: ty::Unevaluated<'tcx, ()>, span: Option<Span>, ) -> EvalToValTreeResult<'tcx> { // Cannot resolve `Unevaluated` constants that contain inference @@ -78,7 +78,7 @@ impl<'tcx> TyCtxt<'tcx> { match ty::Instance::resolve_opt_const_arg(self, param_env, ct.def, ct.substs) { Ok(Some(instance)) => { - let cid = GlobalId { instance, promoted: ct.promoted }; + let cid = GlobalId { instance, promoted: None }; self.const_eval_global_id_for_typeck(param_env, cid, span) } Ok(None) => Err(ErrorHandled::TooGeneric), diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index 834c114ee..ac5fddb7a 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -8,7 +8,7 @@ use rustc_apfloat::{ use rustc_macros::HashStable; use rustc_target::abi::{HasDataLayout, Size}; -use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt}; +use crate::ty::{ParamEnv, ScalarInt, Ty, TyCtxt}; use super::{ AllocId, AllocRange, ConstAllocation, InterpResult, Pointer, PointerArithmetic, Provenance, @@ -27,7 +27,7 @@ pub struct ConstAlloc<'tcx> { /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for /// array length computations, enum discriminants and the pattern matching logic. #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)] -#[derive(HashStable)] +#[derive(HashStable, Lift)] pub enum ConstValue<'tcx> { /// Used only for types with `layout::abi::Scalar` ABI. /// @@ -53,22 +53,6 @@ pub enum ConstValue<'tcx> { #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] static_assert_size!(ConstValue<'_>, 32); -impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> { - type Lifted = ConstValue<'tcx>; - fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ConstValue<'tcx>> { - Some(match self { - ConstValue::Scalar(s) => ConstValue::Scalar(s), - ConstValue::ZeroSized => ConstValue::ZeroSized, - ConstValue::Slice { data, start, end } => { - ConstValue::Slice { data: tcx.lift(data)?, start, end } - } - ConstValue::ByRef { alloc, offset } => { - ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset } - } - }) - } -} - impl<'tcx> ConstValue<'tcx> { #[inline] pub fn try_to_scalar(&self) -> Option<Scalar<AllocId>> { @@ -79,7 +63,7 @@ impl<'tcx> ConstValue<'tcx> { } pub fn try_to_scalar_int(&self) -> Option<ScalarInt> { - Some(self.try_to_scalar()?.assert_int()) + self.try_to_scalar()?.try_to_int().ok() } pub fn try_to_bits(&self, size: Size) -> Option<u128> { @@ -130,9 +114,7 @@ pub enum Scalar<Prov = AllocId> { /// The raw bytes of a simple value. Int(ScalarInt), - /// A pointer into an `Allocation`. An `Allocation` in the `memory` module has a list of - /// relocations, but a `Scalar` is only large enough to contain one, so we just represent the - /// relocation and its associated offset together as a `Pointer` here. + /// A pointer. /// /// We also store the size of the pointer, such that a `Scalar` always knows how big it is. /// The size is always the pointer size of the current target, but this is not information @@ -368,6 +350,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> { } #[inline(always)] + #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) pub fn assert_int(self) -> ScalarInt { self.try_to_int().unwrap() } @@ -389,6 +372,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> { } #[inline(always)] + #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) pub fn assert_bits(self, target_size: Size) -> u128 { self.to_bits(target_size).unwrap() } @@ -502,145 +486,12 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> { } } -#[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, HashStable, Hash)] -pub enum ScalarMaybeUninit<Prov = AllocId> { - Scalar(Scalar<Prov>), - Uninit, -} - -#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -static_assert_size!(ScalarMaybeUninit, 24); - -impl<Prov> From<Scalar<Prov>> for ScalarMaybeUninit<Prov> { - #[inline(always)] - fn from(s: Scalar<Prov>) -> Self { - ScalarMaybeUninit::Scalar(s) - } -} - -// We want the `Debug` output to be readable as it is used by `derive(Debug)` for -// all the Miri types. -impl<Prov: Provenance> fmt::Debug for ScalarMaybeUninit<Prov> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ScalarMaybeUninit::Uninit => write!(f, "<uninitialized>"), - ScalarMaybeUninit::Scalar(s) => write!(f, "{:?}", s), - } - } -} - -impl<Prov: Provenance> fmt::LowerHex for ScalarMaybeUninit<Prov> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ScalarMaybeUninit::Uninit => write!(f, "uninitialized bytes"), - ScalarMaybeUninit::Scalar(s) => write!(f, "{:x}", s), - } - } -} - -impl<Prov> ScalarMaybeUninit<Prov> { - #[inline] - pub fn from_pointer(ptr: Pointer<Prov>, cx: &impl HasDataLayout) -> Self { - ScalarMaybeUninit::Scalar(Scalar::from_pointer(ptr, cx)) - } - - #[inline] - pub fn from_maybe_pointer(ptr: Pointer<Option<Prov>>, cx: &impl HasDataLayout) -> Self { - ScalarMaybeUninit::Scalar(Scalar::from_maybe_pointer(ptr, cx)) - } - - #[inline] - pub fn check_init<'tcx>(self) -> InterpResult<'tcx, Scalar<Prov>> { - match self { - ScalarMaybeUninit::Scalar(scalar) => Ok(scalar), - ScalarMaybeUninit::Uninit => throw_ub!(InvalidUninitBytes(None)), - } - } -} - -impl<'tcx, Prov: Provenance> ScalarMaybeUninit<Prov> { - #[inline(always)] - pub fn to_pointer(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, Pointer<Option<Prov>>> { - self.check_init()?.to_pointer(cx) - } - - #[inline(always)] - pub fn to_bool(self) -> InterpResult<'tcx, bool> { - self.check_init()?.to_bool() - } - - #[inline(always)] - pub fn to_char(self) -> InterpResult<'tcx, char> { - self.check_init()?.to_char() - } - - #[inline(always)] - pub fn to_f32(self) -> InterpResult<'tcx, Single> { - self.check_init()?.to_f32() - } - - #[inline(always)] - pub fn to_f64(self) -> InterpResult<'tcx, Double> { - self.check_init()?.to_f64() - } - - #[inline(always)] - pub fn to_u8(self) -> InterpResult<'tcx, u8> { - self.check_init()?.to_u8() - } - - #[inline(always)] - pub fn to_u16(self) -> InterpResult<'tcx, u16> { - self.check_init()?.to_u16() - } - - #[inline(always)] - pub fn to_u32(self) -> InterpResult<'tcx, u32> { - self.check_init()?.to_u32() - } - - #[inline(always)] - pub fn to_u64(self) -> InterpResult<'tcx, u64> { - self.check_init()?.to_u64() - } - - #[inline(always)] - pub fn to_machine_usize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> { - self.check_init()?.to_machine_usize(cx) - } - - #[inline(always)] - pub fn to_i8(self) -> InterpResult<'tcx, i8> { - self.check_init()?.to_i8() - } - - #[inline(always)] - pub fn to_i16(self) -> InterpResult<'tcx, i16> { - self.check_init()?.to_i16() - } - - #[inline(always)] - pub fn to_i32(self) -> InterpResult<'tcx, i32> { - self.check_init()?.to_i32() - } - - #[inline(always)] - pub fn to_i64(self) -> InterpResult<'tcx, i64> { - self.check_init()?.to_i64() - } - - #[inline(always)] - pub fn to_machine_isize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, i64> { - self.check_init()?.to_machine_isize(cx) - } -} - /// Gets the bytes of a constant slice value. pub fn get_slice_bytes<'tcx>(cx: &impl HasDataLayout, val: ConstValue<'tcx>) -> &'tcx [u8] { if let ConstValue::Slice { data, start, end } = val { let len = end - start; data.inner() - .get_bytes( + .get_bytes_strip_provenance( cx, AllocRange { start: Size::from_bytes(start), size: Size::from_bytes(len) }, ) |