Merging upstream version 1.71.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 114 insertions, 218 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 1b69862ce..b1fda6ff2 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -2,8 +2,8 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
-use cranelift_codegen::ir::{InstructionData, Opcode};
 
 fn codegen_field<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -214,17 +214,7 @@ impl<'tcx> CValue<'tcx> {
     ) -> CValue<'tcx> {
         let layout = self.1;
         match self.0 {
-            CValueInner::ByVal(val) => match layout.abi {
-                Abi::Vector { element: _, count } => {
-                    let count = u8::try_from(count).expect("SIMD type with more than 255 lanes???");
-                    let field = u8::try_from(field.index()).unwrap();
-                    assert!(field < count);
-                    let lane = fx.bcx.ins().extractlane(val, field);
-                    let field_layout = layout.field(&*fx, usize::from(field));
-                    CValue::by_val(lane, field_layout)
-                }
-                _ => unreachable!("value_field for ByVal with abi {:?}", layout.abi),
-            },
+            CValueInner::ByVal(_) => unreachable!(),
             CValueInner::ByValPair(val1, val2) => match layout.abi {
                 Abi::ScalarPair(_, _) => {
                     let val = match field.as_u32() {
@@ -258,16 +248,7 @@ impl<'tcx> CValue<'tcx> {
         let lane_layout = fx.layout_of(lane_ty);
         assert!(lane_idx < lane_count);
         match self.0 {
-            CValueInner::ByVal(val) => match layout.abi {
-                Abi::Vector { element: _, count: _ } => {
-                    assert!(lane_count <= u8::MAX.into(), "SIMD type with more than 255 lanes???");
-                    let lane_idx = u8::try_from(lane_idx).unwrap();
-                    let lane = fx.bcx.ins().extractlane(val, lane_idx);
-                    CValue::by_val(lane, lane_layout)
-                }
-                _ => unreachable!("value_lane for ByVal with abi {:?}", layout.abi),
-            },
-            CValueInner::ByValPair(_, _) => unreachable!(),
+            CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(),
             CValueInner::ByRef(ptr, None) => {
                 let field_offset = lane_layout.size * lane_idx;
                 let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap());
@@ -277,14 +258,6 @@ impl<'tcx> CValue<'tcx> {
         }
     }
 
-    pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
-        crate::unsize::coerce_unsized_into(fx, self, dest);
-    }
-
-    pub(crate) fn coerce_dyn_star(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
-        crate::unsize::coerce_dyn_star(fx, self, dest);
-    }
-
     /// If `ty` is signed, `const_val` must already be sign extended.
     pub(crate) fn const_val(
         fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -345,10 +318,9 @@ pub(crate) struct CPlace<'tcx> {
 }
 
 #[derive(Debug, Copy, Clone)]
-pub(crate) enum CPlaceInner {
+enum CPlaceInner {
     Var(Local, Variable),
     VarPair(Local, Variable, Variable),
-    VarLane(Local, Variable, u8),
     Addr(Pointer, Option<Value>),
 }
 
@@ -357,10 +329,6 @@ impl<'tcx> CPlace<'tcx> {
         self.layout
     }
 
-    pub(crate) fn inner(&self) -> &CPlaceInner {
-        &self.inner
-    }
-
     pub(crate) fn new_stack_slot(
         fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
@@ -376,7 +344,7 @@ impl<'tcx> CPlace<'tcx> {
         if layout.size.bytes() >= u64::from(u32::MAX - 16) {
             fx.tcx
                 .sess
-                .fatal(&format!("values of type {} are too big to store on the stack", layout.ty));
+                .fatal(format!("values of type {} are too big to store on the stack", layout.ty));
         }
 
         let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
@@ -442,12 +410,6 @@ impl<'tcx> CPlace<'tcx> {
                 //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
                 CValue::by_val_pair(val1, val2, layout)
             }
-            CPlaceInner::VarLane(_local, var, lane) => {
-                let val = fx.bcx.use_var(var);
-                //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                let val = fx.bcx.ins().extractlane(val, lane);
-                CValue::by_val(val, layout)
-            }
             CPlaceInner::Addr(ptr, extra) => {
                 if let Some(extra) = extra {
                     CValue::by_ref_unsized(ptr, extra, layout)
@@ -458,21 +420,56 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
+    pub(crate) fn debug_comment(self) -> (&'static str, String) {
+        match self.inner {
+            CPlaceInner::Var(_local, var) => ("ssa", format!("var={}", var.index())),
+            CPlaceInner::VarPair(_local, var1, var2) => {
+                ("ssa", format!("var=({}, {})", var1.index(), var2.index()))
+            }
+            CPlaceInner::Addr(ptr, meta) => {
+                let meta =
+                    if let Some(meta) = meta { format!(",meta={}", meta) } else { String::new() };
+                match ptr.debug_base_and_offset() {
+                    (crate::pointer::PointerBase::Addr(addr), offset) => {
+                        ("reuse", format!("storage={}{}{}", addr, offset, meta))
+                    }
+                    (crate::pointer::PointerBase::Stack(stack_slot), offset) => {
+                        ("stack", format!("storage={}{}{}", stack_slot, offset, meta))
+                    }
+                    (crate::pointer::PointerBase::Dangling(align), offset) => {
+                        ("zst", format!("align={},offset={}", align.bytes(), offset))
+                    }
+                }
+            }
+        }
+    }
+
     #[track_caller]
     pub(crate) fn to_ptr(self) -> Pointer {
-        match self.to_ptr_maybe_unsized() {
-            (ptr, None) => ptr,
-            (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self),
+        match self.inner {
+            CPlaceInner::Addr(ptr, None) => ptr,
+            CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self),
+            CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => {
+                bug!("Expected CPlace::Addr, found {:?}", self)
+            }
         }
     }
 
     #[track_caller]
-    pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) {
+    pub(crate) fn to_ptr_unsized(self) -> (Pointer, Value) {
         match self.inner {
-            CPlaceInner::Addr(ptr, extra) => (ptr, extra),
-            CPlaceInner::Var(_, _)
-            | CPlaceInner::VarPair(_, _, _)
-            | CPlaceInner::VarLane(_, _, _) => bug!("Expected CPlace::Addr, found {:?}", self),
+            CPlaceInner::Addr(ptr, Some(extra)) => (ptr, extra),
+            CPlaceInner::Addr(_, None) | CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => {
+                bug!("Expected unsized cplace, found {:?}", self)
+            }
+        }
+    }
+
+    pub(crate) fn try_to_ptr(self) -> Option<Pointer> {
+        match self.inner {
+            CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => None,
+            CPlaceInner::Addr(ptr, None) => Some(ptr),
+            CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self),
         }
     }
 
@@ -496,7 +493,7 @@ impl<'tcx> CPlace<'tcx> {
         from: CValue<'tcx>,
         method: &'static str,
     ) {
-        fn transmute_value<'tcx>(
+        fn transmute_scalar<'tcx>(
             fx: &mut FunctionCx<'_, '_, 'tcx>,
             var: Variable,
             data: Value,
@@ -520,7 +517,7 @@ impl<'tcx> CPlace<'tcx> {
                 | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
                 _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
                 _ if src_ty.is_vector() || dst_ty.is_vector() => {
-                    // FIXME do something more efficient for transmutes between vectors and integers.
+                    // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
                     let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
                         kind: StackSlotKind::ExplicitSlot,
                         // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
@@ -554,7 +551,7 @@ impl<'tcx> CPlace<'tcx> {
                 format!(
                     "{}: {:?}: {:?} <- {:?}: {:?}",
                     method,
-                    self.inner(),
+                    self.inner,
                     self.layout().ty,
                     from.0,
                     from.layout().ty
@@ -563,32 +560,11 @@ impl<'tcx> CPlace<'tcx> {
         }
 
         let dst_layout = self.layout();
-        let to_ptr = match self.inner {
+        match self.inner {
             CPlaceInner::Var(_local, var) => {
-                if let ty::Array(element, len) = dst_layout.ty.kind() {
-                    // Can only happen for vector types
-                    let len = u32::try_from(len.eval_target_usize(fx.tcx, ParamEnv::reveal_all()))
-                        .unwrap();
-                    let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
-
-                    let data = match from.0 {
-                        CValueInner::ByRef(ptr, None) => {
-                            let mut flags = MemFlags::new();
-                            flags.set_notrap();
-                            ptr.load(fx, vector_ty, flags)
-                        }
-                        CValueInner::ByVal(_)
-                        | CValueInner::ByValPair(_, _)
-                        | CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"),
-                    };
-
-                    fx.bcx.def_var(var, data);
-                    return;
-                }
                 let data = CValue(from.0, dst_layout).load_scalar(fx);
                 let dst_ty = fx.clif_type(self.layout().ty).unwrap();
-                transmute_value(fx, var, data, dst_ty);
-                return;
+                transmute_scalar(fx, var, data, dst_ty);
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
                 let (data1, data2) = if from.layout().ty == dst_layout.ty {
@@ -599,80 +575,61 @@ impl<'tcx> CPlace<'tcx> {
                     CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
                 };
                 let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
-                transmute_value(fx, var1, data1, dst_ty1);
-                transmute_value(fx, var2, data2, dst_ty2);
-                return;
+                transmute_scalar(fx, var1, data1, dst_ty1);
+                transmute_scalar(fx, var2, data2, dst_ty2);
             }
-            CPlaceInner::VarLane(_local, var, lane) => {
-                let data = from.load_scalar(fx);
-
-                // First get the old vector
-                let vector = fx.bcx.use_var(var);
-                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
-
-                // Next insert the written lane into the vector
-                let vector = fx.bcx.ins().insertlane(vector, data, lane);
-
-                // Finally write the new vector
-                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                fx.bcx.def_var(var, vector);
-
-                return;
-            }
-            CPlaceInner::Addr(ptr, None) => {
+            CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self),
+            CPlaceInner::Addr(to_ptr, None) => {
                 if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited {
                     return;
                 }
-                ptr
-            }
-            CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self),
-        };
 
-        let mut flags = MemFlags::new();
-        flags.set_notrap();
-        match from.layout().abi {
-            // FIXME make Abi::Vector work too
-            Abi::Scalar(_) => {
-                let val = from.load_scalar(fx);
-                to_ptr.store(fx, val, flags);
-                return;
-            }
-            Abi::ScalarPair(a_scalar, b_scalar) => {
-                let (value, extra) = from.load_scalar_pair(fx);
-                let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
-                to_ptr.store(fx, value, flags);
-                to_ptr.offset(fx, b_offset).store(fx, extra, flags);
-                return;
-            }
-            _ => {}
-        }
+                let mut flags = MemFlags::new();
+                flags.set_notrap();
+                match from.layout().abi {
+                    Abi::Scalar(_) => {
+                        let val = from.load_scalar(fx);
+                        to_ptr.store(fx, val, flags);
+                        return;
+                    }
+                    Abi::ScalarPair(a_scalar, b_scalar) => {
+                        let (value, extra) = from.load_scalar_pair(fx);
+                        let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+                        to_ptr.store(fx, value, flags);
+                        to_ptr.offset(fx, b_offset).store(fx, extra, flags);
+                        return;
+                    }
+                    _ => {}
+                }
 
-        match from.0 {
-            CValueInner::ByVal(val) => {
-                to_ptr.store(fx, val, flags);
-            }
-            CValueInner::ByValPair(_, _) => {
-                bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
-            }
-            CValueInner::ByRef(from_ptr, None) => {
-                let from_addr = from_ptr.get_addr(fx);
-                let to_addr = to_ptr.get_addr(fx);
-                let src_layout = from.1;
-                let size = dst_layout.size.bytes();
-                let src_align = src_layout.align.abi.bytes() as u8;
-                let dst_align = dst_layout.align.abi.bytes() as u8;
-                fx.bcx.emit_small_memory_copy(
-                    fx.target_config,
-                    to_addr,
-                    from_addr,
-                    size,
-                    dst_align,
-                    src_align,
-                    true,
-                    flags,
-                );
+                match from.0 {
+                    CValueInner::ByVal(val) => {
+                        to_ptr.store(fx, val, flags);
+                    }
+                    CValueInner::ByValPair(_, _) => {
+                        bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
+                    }
+                    CValueInner::ByRef(from_ptr, None) => {
+                        let from_addr = from_ptr.get_addr(fx);
+                        let to_addr = to_ptr.get_addr(fx);
+                        let src_layout = from.1;
+                        let size = dst_layout.size.bytes();
+                        let src_align = src_layout.align.abi.bytes() as u8;
+                        let dst_align = dst_layout.align.abi.bytes() as u8;
+                        fx.bcx.emit_small_memory_copy(
+                            fx.target_config,
+                            to_addr,
+                            from_addr,
+                            size,
+                            dst_align,
+                            src_align,
+                            true,
+                            flags,
+                        );
+                    }
+                    CValueInner::ByRef(_, Some(_)) => todo!(),
+                }
             }
-            CValueInner::ByRef(_, Some(_)) => todo!(),
         }
     }
 
@@ -692,40 +649,6 @@ impl<'tcx> CPlace<'tcx> {
         let layout = self.layout();
 
         match self.inner {
-            CPlaceInner::Var(local, var) => match layout.ty.kind() {
-                ty::Array(_, _) => {
-                    // Can only happen for vector types
-                    return CPlace {
-                        inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()),
-                        layout: layout.field(fx, field.as_u32().try_into().unwrap()),
-                    };
-                }
-                ty::Adt(adt_def, substs) if layout.ty.is_simd() => {
-                    let f0 = &adt_def.non_enum_variant().fields[FieldIdx::from_u32(0)];
-                    let f0_ty = f0.ty(fx.tcx, substs);
-
-                    match f0_ty.kind() {
-                        ty::Array(_, _) => {
-                            assert_eq!(field.as_u32(), 0);
-                            return CPlace {
-                                inner: CPlaceInner::Var(local, var),
-                                layout: layout.field(fx, field.as_u32().try_into().unwrap()),
-                            };
-                        }
-                        _ => {
-                            return CPlace {
-                                inner: CPlaceInner::VarLane(
-                                    local,
-                                    var,
-                                    field.as_u32().try_into().unwrap(),
-                                ),
-                                layout: layout.field(fx, field.as_u32().try_into().unwrap()),
-                            };
-                        }
-                    }
-                }
-                _ => {}
-            },
             CPlaceInner::VarPair(local, var1, var2) => {
                 let layout = layout.field(&*fx, field.index());
 
@@ -738,7 +661,12 @@ impl<'tcx> CPlace<'tcx> {
             _ => {}
         }
 
-        let (base, extra) = self.to_ptr_maybe_unsized();
+        let (base, extra) = match self.inner {
+            CPlaceInner::Addr(ptr, extra) => (ptr, extra),
+            CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => {
+                bug!("Expected CPlace::Addr, found {:?}", self)
+            }
+        };
 
         let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field);
         if field_layout.is_unsized() {
@@ -767,15 +695,8 @@ impl<'tcx> CPlace<'tcx> {
         assert!(lane_idx < lane_count);
 
         match self.inner {
-            CPlaceInner::Var(local, var) => {
-                assert!(matches!(layout.abi, Abi::Vector { .. }));
-                CPlace {
-                    inner: CPlaceInner::VarLane(local, var, lane_idx.try_into().unwrap()),
-                    layout: lane_layout,
-                }
-            }
+            CPlaceInner::Var(_, _) => unreachable!(),
             CPlaceInner::VarPair(_, _, _) => unreachable!(),
-            CPlaceInner::VarLane(_, _, _) => unreachable!(),
             CPlaceInner::Addr(ptr, None) => {
                 let field_offset = lane_layout.size * lane_idx;
                 let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap());
@@ -794,34 +715,13 @@ impl<'tcx> CPlace<'tcx> {
             ty::Array(elem_ty, _) => {
                 let elem_layout = fx.layout_of(*elem_ty);
                 match self.inner {
-                    CPlaceInner::Var(local, var) => {
-                        // This is a hack to handle `vector_val.0[1]`. It doesn't allow dynamic
-                        // indexing.
-                        let lane_idx = match fx.bcx.func.dfg.insts
-                            [fx.bcx.func.dfg.value_def(index).unwrap_inst()]
-                        {
-                            InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => imm,
-                            _ => bug!(
-                                "Dynamic indexing into a vector type is not supported: {self:?}[{index}]"
-                            ),
-                        };
-                        return CPlace {
-                            inner: CPlaceInner::VarLane(
-                                local,
-                                var,
-                                lane_idx.bits().try_into().unwrap(),
-                            ),
-                            layout: elem_layout,
-                        };
-                    }
                     CPlaceInner::Addr(addr, None) => (elem_layout, addr),
-                    CPlaceInner::Addr(_, Some(_))
-                    | CPlaceInner::VarPair(_, _, _)
-                    | CPlaceInner::VarLane(_, _, _) => bug!("Can't index into {self:?}"),
+                    CPlaceInner::Var(_, _)
+                    | CPlaceInner::Addr(_, Some(_))
+                    | CPlaceInner::VarPair(_, _, _) => bug!("Can't index into {self:?}"),
                 }
-                // FIXME use VarLane in case of Var with simd type
             }
-            ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0),
+            ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_unsized().0),
             _ => bug!("place_index({:?})", self.layout().ty),
         };
 
@@ -846,12 +746,8 @@ impl<'tcx> CPlace<'tcx> {
         layout: TyAndLayout<'tcx>,
     ) -> CValue<'tcx> {
         if has_ptr_meta(fx.tcx, self.layout().ty) {
-            let (ptr, extra) = self.to_ptr_maybe_unsized();
-            CValue::by_val_pair(
-                ptr.get_addr(fx),
-                extra.expect("unsized type without metadata"),
-                layout,
-            )
+            let (ptr, extra) = self.to_ptr_unsized();
+            CValue::by_val_pair(ptr.get_addr(fx), extra, layout)
         } else {
             CValue::by_val(self.to_ptr().get_addr(fx), layout)
         }