diff options
Diffstat (limited to 'compiler/rustc_abi/src')
| -rw-r--r-- | compiler/rustc_abi/src/layout.rs | 93 | ||||
| -rw-r--r-- | compiler/rustc_abi/src/lib.rs | 80 |
2 files changed, 108 insertions, 65 deletions
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 54858b520..c863acde7 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -8,19 +8,6 @@ use rand_xoshiro::Xoshiro128StarStar; use tracing::debug; -// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`. -// This is used to go between `memory_index` (source field order to memory order) -// and `inverse_memory_index` (memory order to source field order). -// See also `FieldsShape::Arbitrary::memory_index` for more details. -// FIXME(eddyb) build a better abstraction for permutations, if possible. -fn invert_mapping(map: &[u32]) -> Vec<u32> { - let mut inverse = vec![0; map.len()]; - for i in 0..map.len() { - inverse[map[i] as usize] = i as u32; - } - inverse -} - pub trait LayoutCalculator { type TargetDataLayoutRef: Borrow<TargetDataLayout>; @@ -43,10 +30,10 @@ pub trait LayoutCalculator { .max_by_key(|niche| niche.available(dl)); LayoutS { - variants: Variants::Single { index: VariantIdx::new(0) }, + variants: Variants::Single { index: FIRST_VARIANT }, fields: FieldsShape::Arbitrary { - offsets: vec![Size::ZERO, b_offset], - memory_index: vec![0, 1], + offsets: [Size::ZERO, b_offset].into(), + memory_index: [0, 1].into(), }, abi: Abi::ScalarPair(a, b), largest_niche, @@ -58,18 +45,18 @@ pub trait LayoutCalculator { fn univariant( &self, dl: &TargetDataLayout, - fields: &[Layout<'_>], + fields: &IndexSlice<FieldIdx, Layout<'_>>, repr: &ReprOptions, kind: StructKind, ) -> Option<LayoutS> { let pack = repr.pack; let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align }; - let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect(); + let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect(); let optimize = !repr.inhibit_struct_field_reordering_opt(); if optimize { let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() }; - let optimizing = &mut inverse_memory_index[..end]; + let optimizing = &mut inverse_memory_index.raw[..end]; let effective_field_align = |layout: Layout<'_>| { if let Some(pack) = pack { // return the packed alignment in bytes @@ -105,7 +92,7 @@ pub trait LayoutCalculator { // Place ZSTs first to avoid "interesting offsets", // especially with only one or two non-ZST fields. // Then place largest alignments first, largest niches within an alignment group last - let f = fields[x as usize]; + let f = fields[x]; let niche_size = f.largest_niche().map_or(0, |n| n.available(dl)); (!f.0.is_zst(), cmp::Reverse(effective_field_align(f)), niche_size) }); @@ -117,7 +104,7 @@ pub trait LayoutCalculator { // And put the largest niche in an alignment group at the end // so it can be used as discriminant in jagged enums optimizing.sort_by_key(|&x| { - let f = fields[x as usize]; + let f = fields[x]; let niche_size = f.largest_niche().map_or(0, |n| n.available(dl)); (effective_field_align(f), niche_size) }); @@ -135,7 +122,7 @@ pub trait LayoutCalculator { // At the bottom of this function, we invert `inverse_memory_index` to // produce `memory_index` (see `invert_mapping`). let mut sized = true; - let mut offsets = vec![Size::ZERO; fields.len()]; + let mut offsets = IndexVec::from_elem(Size::ZERO, &fields); let mut offset = Size::ZERO; let mut largest_niche = None; let mut largest_niche_available = 0; @@ -146,7 +133,7 @@ pub trait LayoutCalculator { offset = prefix_size.align_to(prefix_align); } for &i in &inverse_memory_index { - let field = &fields[i as usize]; + let field = &fields[i]; if !sized { self.delay_bug(&format!( "univariant: field #{} comes after unsized field", @@ -168,7 +155,7 @@ pub trait LayoutCalculator { align = align.max(field_align); debug!("univariant offset: {:?} field: {:#?}", offset, field); - offsets[i as usize] = offset; + offsets[i] = offset; if let Some(mut niche) = field.largest_niche() { let available = niche.available(dl); @@ -192,14 +179,18 @@ pub trait LayoutCalculator { // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0. // Field 5 would be the first element, so memory_index is i: // Note: if we didn't optimize, it's already right. - let memory_index = - if optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index }; + let memory_index = if optimize { + inverse_memory_index.invert_bijective_mapping() + } else { + debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices())); + inverse_memory_index.into_iter().map(FieldIdx::as_u32).collect() + }; let size = min_size.align_to(align.abi); let mut abi = Abi::Aggregate { sized }; // Unpack newtype ABIs and find scalar pairs. if sized && size.bytes() > 0 { // All other fields must be ZSTs. - let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.0.is_zst()); + let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.0.is_zst()); match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) { // We have exactly one non-ZST field. @@ -238,13 +229,13 @@ pub trait LayoutCalculator { let pair = self.scalar_pair(a, b); let pair_offsets = match pair.fields { FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index, &[0, 1]); + assert_eq!(memory_index.raw, [0, 1]); offsets } _ => panic!(), }; - if offsets[i] == pair_offsets[0] - && offsets[j] == pair_offsets[1] + if offsets[i] == pair_offsets[FieldIdx::from_usize(0)] + && offsets[j] == pair_offsets[FieldIdx::from_usize(1)] && align == pair.align && size == pair.size { @@ -264,7 +255,7 @@ pub trait LayoutCalculator { abi = Abi::Uninhabited; } Some(LayoutS { - variants: Variants::Single { index: VariantIdx::new(0) }, + variants: Variants::Single { index: FIRST_VARIANT }, fields: FieldsShape::Arbitrary { offsets, memory_index }, abi, largest_niche, @@ -277,7 +268,7 @@ pub trait LayoutCalculator { let dl = self.current_data_layout(); let dl = dl.borrow(); LayoutS { - variants: Variants::Single { index: VariantIdx::new(0) }, + variants: Variants::Single { index: FIRST_VARIANT }, fields: FieldsShape::Primitive, abi: Abi::Uninhabited, largest_niche: None, @@ -289,7 +280,7 @@ pub trait LayoutCalculator { fn layout_of_struct_or_enum( &self, repr: &ReprOptions, - variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>, + variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, Layout<'_>>>, is_enum: bool, is_unsafe_cell: bool, scalar_valid_range: (Bound<u128>, Bound<u128>), @@ -312,7 +303,7 @@ pub trait LayoutCalculator { // but *not* an encoding of the discriminant (e.g., a tag value). // See issue #49298 for more details on the need to leave space // for non-ZST uninhabited data (mostly partial initialization). - let absent = |fields: &[Layout<'_>]| { + let absent = |fields: &IndexSlice<FieldIdx, Layout<'_>>| { let uninhabited = fields.iter().any(|f| f.abi().is_uninhabited()); let is_zst = fields.iter().all(|f| f.0.is_zst()); uninhabited && is_zst @@ -331,7 +322,7 @@ pub trait LayoutCalculator { } // If it's a struct, still compute a layout so that we can still compute the // field offsets. - None => VariantIdx::new(0), + None => FIRST_VARIANT, }; let is_struct = !is_enum || @@ -467,7 +458,7 @@ pub trait LayoutCalculator { .max_by_key(|(_i, layout)| layout.size.bytes()) .map(|(i, _layout)| i)?; - let all_indices = (0..=variants.len() - 1).map(VariantIdx::new); + let all_indices = variants.indices(); let needs_disc = |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap().index() @@ -510,7 +501,7 @@ pub trait LayoutCalculator { // It'll fit, but we need to make some adjustments. match layout.fields { FieldsShape::Arbitrary { ref mut offsets, .. } => { - for (j, offset) in offsets.iter_mut().enumerate() { + for (j, offset) in offsets.iter_enumerated_mut() { if !variants[i][j].0.is_zst() { *offset += this_offset; } @@ -577,8 +568,8 @@ pub trait LayoutCalculator { variants: IndexVec::new(), }, fields: FieldsShape::Arbitrary { - offsets: vec![niche_offset], - memory_index: vec![0], + offsets: [niche_offset].into(), + memory_index: [0].into(), }, abi, largest_niche, @@ -651,7 +642,8 @@ pub trait LayoutCalculator { st.variants = Variants::Single { index: i }; // Find the first field we can't move later // to make room for a larger discriminant. - for field in st.fields.index_by_increasing_offset().map(|j| &field_layouts[j]) { + for field_idx in st.fields.index_by_increasing_offset() { + let field = &field_layouts[FieldIdx::from_usize(field_idx)]; if !field.0.is_zst() || field.align().abi.bytes() != 1 { start_align = start_align.min(field.align().abi); break; @@ -802,13 +794,13 @@ pub trait LayoutCalculator { let pair = self.scalar_pair(tag, prim_scalar); let pair_offsets = match pair.fields { FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index, &[0, 1]); + assert_eq!(memory_index.raw, [0, 1]); offsets } _ => panic!(), }; - if pair_offsets[0] == Size::ZERO - && pair_offsets[1] == *offset + if pair_offsets[FieldIdx::from_u32(0)] == Size::ZERO + && pair_offsets[FieldIdx::from_u32(1)] == *offset && align == pair.align && size == pair.size { @@ -844,7 +836,10 @@ pub trait LayoutCalculator { tag_field: 0, variants: IndexVec::new(), }, - fields: FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + memory_index: [0].into(), + }, largest_niche, abi, align, @@ -883,7 +878,7 @@ pub trait LayoutCalculator { fn layout_of_union( &self, repr: &ReprOptions, - variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>, + variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, Layout<'_>>>, ) -> Option<LayoutS> { let dl = self.current_data_layout(); let dl = dl.borrow(); @@ -896,8 +891,8 @@ pub trait LayoutCalculator { let optimize = !repr.inhibit_union_abi_opt(); let mut size = Size::ZERO; let mut abi = Abi::Aggregate { sized: true }; - let index = VariantIdx::new(0); - for field in &variants[index] { + let only_variant = &variants[FIRST_VARIANT]; + for field in only_variant { assert!(field.0.is_sized()); align = align.max(field.align()); @@ -930,8 +925,8 @@ pub trait LayoutCalculator { } Some(LayoutS { - variants: Variants::Single { index }, - fields: FieldsShape::Union(NonZeroUsize::new(variants[index].len())?), + variants: Variants::Single { index: FIRST_VARIANT }, + fields: FieldsShape::Union(NonZeroUsize::new(only_variant.len())?), abi, largest_niche: None, align, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 39574ca55..b0c0ee942 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -11,7 +11,7 @@ use bitflags::bitflags; use rustc_data_structures::intern::Interned; #[cfg(feature = "nightly")] use rustc_data_structures::stable_hasher::StableOrd; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::{Idx, IndexSlice, IndexVec}; #[cfg(feature = "nightly")] use rustc_macros::HashStable_Generic; #[cfg(feature = "nightly")] @@ -1057,6 +1057,32 @@ impl Scalar { } } +rustc_index::newtype_index! { + /// The *source-order* index of a field in a variant. + /// + /// This is how most code after type checking refers to fields, rather than + /// using names (as names have hygiene complications and more complex lookup). + /// + /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order. + /// (It is for `repr(C)` `struct`s, however.) + /// + /// For example, in the following types, + /// ```rust + /// # enum Never {} + /// # #[repr(u16)] + /// enum Demo1 { + /// Variant0 { a: Never, b: i32 } = 100, + /// Variant1 { c: u8, d: u64 } = 10, + /// } + /// struct Demo2 { e: u8, f: u16, g: u8 } + /// ``` + /// `b` is `FieldIdx(1)` in `VariantIdx(0)`, + /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and + /// `f` is `FieldIdx(1)` in `VariantIdx(0)`. + #[derive(HashStable_Generic)] + pub struct FieldIdx {} +} + /// Describes how the fields of a type are located in memory. #[derive(PartialEq, Eq, Hash, Clone, Debug)] #[cfg_attr(feature = "nightly", derive(HashStable_Generic))] @@ -1082,7 +1108,7 @@ pub enum FieldsShape { /// ordered to match the source definition order. /// This vector does not go in increasing order. // FIXME(eddyb) use small vector optimization for the common case. - offsets: Vec<Size>, + offsets: IndexVec<FieldIdx, Size>, /// Maps source order field indices to memory order indices, /// depending on how the fields were reordered (if at all). @@ -1096,7 +1122,7 @@ pub enum FieldsShape { /// // FIXME(eddyb) build a better abstraction for permutations, if possible. // FIXME(camlorn) also consider small vector optimization here. - memory_index: Vec<u32>, + memory_index: IndexVec<FieldIdx, u32>, }, } @@ -1131,7 +1157,7 @@ impl FieldsShape { assert!(i < count); stride * i } - FieldsShape::Arbitrary { ref offsets, .. } => offsets[i], + FieldsShape::Arbitrary { ref offsets, .. } => offsets[FieldIdx::from_usize(i)], } } @@ -1142,28 +1168,27 @@ impl FieldsShape { unreachable!("FieldsShape::memory_index: `Primitive`s have no fields") } FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { ref memory_index, .. } => memory_index[i].try_into().unwrap(), + FieldsShape::Arbitrary { ref memory_index, .. } => { + memory_index[FieldIdx::from_usize(i)].try_into().unwrap() + } } } /// Gets source indices of the fields by increasing offsets. #[inline] - pub fn index_by_increasing_offset<'a>(&'a self) -> impl Iterator<Item = usize> + 'a { + pub fn index_by_increasing_offset(&self) -> impl Iterator<Item = usize> + '_ { let mut inverse_small = [0u8; 64]; - let mut inverse_big = vec![]; + let mut inverse_big = IndexVec::new(); let use_small = self.count() <= inverse_small.len(); // We have to write this logic twice in order to keep the array small. if let FieldsShape::Arbitrary { ref memory_index, .. } = *self { if use_small { - for i in 0..self.count() { - inverse_small[memory_index[i] as usize] = i as u8; + for (field_idx, &mem_idx) in memory_index.iter_enumerated() { + inverse_small[mem_idx as usize] = field_idx.as_u32() as u8; } } else { - inverse_big = vec![0; self.count()]; - for i in 0..self.count() { - inverse_big[memory_index[i] as usize] = i as u32; - } + inverse_big = memory_index.invert_bijective_mapping(); } } @@ -1173,7 +1198,7 @@ impl FieldsShape { if use_small { inverse_small[i] as usize } else { - inverse_big[i] as usize + inverse_big[i as u32].as_usize() } } }) @@ -1380,8 +1405,21 @@ impl Niche { } rustc_index::newtype_index! { + /// The *source-order* index of a variant in a type. + /// + /// For enums, these are always `0..variant_count`, regardless of any + /// custom discriminants that may have been defined, and including any + /// variants that may end up uninhabited due to field types. (Some of the + /// variants may not be present in a monomorphized ABI [`Variants`], but + /// those skipped variants are always counted when determining the *index*.) + /// + /// `struct`s, `tuples`, and `unions`s are considered to have a single variant + /// with variant index zero, aka [`FIRST_VARIANT`]. #[derive(HashStable_Generic)] - pub struct VariantIdx {} + pub struct VariantIdx { + /// Equivalent to `VariantIdx(0)`. + const FIRST_VARIANT = 0; + } } #[derive(PartialEq, Eq, Hash, Clone)] @@ -1422,7 +1460,7 @@ impl LayoutS { let size = scalar.size(cx); let align = scalar.align(cx); LayoutS { - variants: Variants::Single { index: VariantIdx::new(0) }, + variants: Variants::Single { index: FIRST_VARIANT }, fields: FieldsShape::Primitive, abi: Abi::Scalar(scalar), largest_niche, @@ -1484,6 +1522,16 @@ impl<'a> Layout<'a> { pub fn size(self) -> Size { self.0.0.size } + + /// Whether the layout is from a type that implements [`std::marker::PointerLike`]. + /// + /// Currently, that means that the type is pointer-sized, pointer-aligned, + /// and has a scalar ABI. + pub fn is_pointer_like(self, data_layout: &TargetDataLayout) -> bool { + self.size() == data_layout.pointer_size + && self.align().abi == data_layout.pointer_align.abi + && matches!(self.abi(), Abi::Scalar(..)) + } } #[derive(Copy, Clone, PartialEq, Eq, Debug)] |