Merging upstream version 1.67.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 640 insertions, 1590 deletions

diff --git a/compiler/rustc_ty_utils/Cargo.toml b/compiler/rustc_ty_utils/Cargo.toml
index 5e4ba4730..52fbd3ae0 100644
--- a/compiler/rustc_ty_utils/Cargo.toml
+++ b/compiler/rustc_ty_utils/Cargo.toml
@@ -4,8 +4,6 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-rand = "0.8.4"
-rand_xoshiro = "0.6.0"
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }
 rustc_data_structures = { path = "../rustc_data_structures" }
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index e057bb668..f8ff31f97 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -1,10 +1,11 @@
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::vec::IndexVec;
 use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
-use rustc_middle::ty::abstract_const::{CastKind, Node, NodeId};
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::thir::visit;
+use rustc_middle::thir::visit::Visitor;
+use rustc_middle::ty::abstract_const::CastKind;
+use rustc_middle::ty::{self, Expr, TyCtxt, TypeVisitable};
 use rustc_middle::{mir, thir};
 use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
@@ -31,10 +32,8 @@ pub(crate) fn destructure_const<'tcx>(
     let (fields, variant) = match const_.ty().kind() {
         ty::Array(inner_ty, _) | ty::Slice(inner_ty) => {
             // construct the consts for the elements of the array/slice
-            let field_consts = branches
-                .iter()
-                .map(|b| tcx.mk_const(ty::ConstS { kind: ty::ConstKind::Value(*b), ty: *inner_ty }))
-                .collect::<Vec<_>>();
+            let field_consts =
+                branches.iter().map(|b| tcx.mk_const(*b, *inner_ty)).collect::<Vec<_>>();
             debug!(?field_consts);
 
             (field_consts, None)
@@ -52,10 +51,7 @@ pub(crate) fn destructure_const<'tcx>(
 
             for (field, field_valtree) in iter::zip(fields, branches) {
                 let field_ty = field.ty(tcx, substs);
-                let field_const = tcx.mk_const(ty::ConstS {
-                    kind: ty::ConstKind::Value(*field_valtree),
-                    ty: field_ty,
-                });
+                let field_const = tcx.mk_const(*field_valtree, field_ty);
                 field_consts.push(field_const);
             }
             debug!(?field_consts);
@@ -64,12 +60,7 @@ pub(crate) fn destructure_const<'tcx>(
         }
         ty::Tuple(elem_tys) => {
             let fields = iter::zip(*elem_tys, branches)
-                .map(|(elem_ty, elem_valtree)| {
-                    tcx.mk_const(ty::ConstS {
-                        kind: ty::ConstKind::Value(*elem_valtree),
-                        ty: elem_ty,
-                    })
-                })
+                .map(|(elem_ty, elem_valtree)| tcx.mk_const(*elem_valtree, elem_ty))
                 .collect::<Vec<_>>();
 
             (fields, None)
@@ -82,328 +73,278 @@ pub(crate) fn destructure_const<'tcx>(
     ty::DestructuredConst { variant, fields }
 }
 
-pub struct AbstractConstBuilder<'a, 'tcx> {
-    tcx: TyCtxt<'tcx>,
-    body_id: thir::ExprId,
-    body: &'a thir::Thir<'tcx>,
-    /// The current WIP node tree.
-    nodes: IndexVec<NodeId, Node<'tcx>>,
-}
-
-impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
-    fn root_span(&self) -> Span {
-        self.body.exprs[self.body_id].span
-    }
-
-    fn error(&mut self, sub: GenericConstantTooComplexSub) -> Result<!, ErrorGuaranteed> {
-        let reported = self.tcx.sess.emit_err(GenericConstantTooComplex {
-            span: self.root_span(),
-            maybe_supported: None,
-            sub,
-        });
-
-        Err(reported)
+/// We do not allow all binary operations in abstract consts, so filter disallowed ones.
+fn check_binop(op: mir::BinOp) -> bool {
+    use mir::BinOp::*;
+    match op {
+        Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Shl | Shr | Eq | Lt | Le | Ne
+        | Ge | Gt => true,
+        Offset => false,
     }
+}
 
-    fn maybe_supported_error(
-        &mut self,
-        sub: GenericConstantTooComplexSub,
-    ) -> Result<!, ErrorGuaranteed> {
-        let reported = self.tcx.sess.emit_err(GenericConstantTooComplex {
-            span: self.root_span(),
-            maybe_supported: Some(()),
-            sub,
-        });
-
-        Err(reported)
+/// While we currently allow all unary operations, we still want to explicitly guard against
+/// future changes here.
+fn check_unop(op: mir::UnOp) -> bool {
+    use mir::UnOp::*;
+    match op {
+        Not | Neg => true,
     }
+}
 
-    #[instrument(skip(tcx, body, body_id), level = "debug")]
-    pub fn new(
-        tcx: TyCtxt<'tcx>,
-        (body, body_id): (&'a thir::Thir<'tcx>, thir::ExprId),
-    ) -> Result<Option<AbstractConstBuilder<'a, 'tcx>>, ErrorGuaranteed> {
-        let builder = AbstractConstBuilder { tcx, body_id, body, nodes: IndexVec::new() };
-
-        struct IsThirPolymorphic<'a, 'tcx> {
-            is_poly: bool,
-            thir: &'a thir::Thir<'tcx>,
+fn recurse_build<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    body: &thir::Thir<'tcx>,
+    node: thir::ExprId,
+    root_span: Span,
+) -> Result<ty::Const<'tcx>, ErrorGuaranteed> {
+    use thir::ExprKind;
+    let node = &body.exprs[node];
+
+    let maybe_supported_error = |a| maybe_supported_error(tcx, a, root_span);
+    let error = |a| error(tcx, a, root_span);
+
+    Ok(match &node.kind {
+        // I dont know if handling of these 3 is correct
+        &ExprKind::Scope { value, .. } => recurse_build(tcx, body, value, root_span)?,
+        &ExprKind::PlaceTypeAscription { source, .. }
+        | &ExprKind::ValueTypeAscription { source, .. } => {
+            recurse_build(tcx, body, source, root_span)?
         }
-
-        use crate::rustc_middle::thir::visit::Visitor;
-        use thir::visit;
-
-        impl<'a, 'tcx> IsThirPolymorphic<'a, 'tcx> {
-            fn expr_is_poly(&mut self, expr: &thir::Expr<'tcx>) -> bool {
-                if expr.ty.has_non_region_param() {
-                    return true;
+        &ExprKind::Literal { lit, neg } => {
+            let sp = node.span;
+            match tcx.at(sp).lit_to_const(LitToConstInput { lit: &lit.node, ty: node.ty, neg }) {
+                Ok(c) => c,
+                Err(LitToConstError::Reported(guar)) => {
+                    tcx.const_error_with_guaranteed(node.ty, guar)
                 }
-
-                match expr.kind {
-                    thir::ExprKind::NamedConst { substs, .. } => substs.has_non_region_param(),
-                    thir::ExprKind::ConstParam { .. } => true,
-                    thir::ExprKind::Repeat { value, count } => {
-                        self.visit_expr(&self.thir()[value]);
-                        count.has_non_region_param()
-                    }
-                    _ => false,
+                Err(LitToConstError::TypeError) => {
+                    bug!("encountered type error in lit_to_const")
                 }
             }
+        }
+        &ExprKind::NonHirLiteral { lit, user_ty: _ } => {
+            let val = ty::ValTree::from_scalar_int(lit);
+            tcx.mk_const(val, node.ty)
+        }
+        &ExprKind::ZstLiteral { user_ty: _ } => {
+            let val = ty::ValTree::zst();
+            tcx.mk_const(val, node.ty)
+        }
+        &ExprKind::NamedConst { def_id, substs, user_ty: _ } => {
+            let uneval = ty::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
+            tcx.mk_const(uneval, node.ty)
+        }
+        ExprKind::ConstParam { param, .. } => tcx.mk_const(*param, node.ty),
 
-            fn pat_is_poly(&mut self, pat: &thir::Pat<'tcx>) -> bool {
-                if pat.ty.has_non_region_param() {
-                    return true;
-                }
+        ExprKind::Call { fun, args, .. } => {
+            let fun = recurse_build(tcx, body, *fun, root_span)?;
 
-                match pat.kind {
-                    thir::PatKind::Constant { value } => value.has_non_region_param(),
-                    thir::PatKind::Range(box thir::PatRange { lo, hi, .. }) => {
-                        lo.has_non_region_param() || hi.has_non_region_param()
-                    }
-                    _ => false,
-                }
+            let mut new_args = Vec::<ty::Const<'tcx>>::with_capacity(args.len());
+            for &id in args.iter() {
+                new_args.push(recurse_build(tcx, body, id, root_span)?);
             }
+            let new_args = tcx.mk_const_list(new_args.iter());
+            tcx.mk_const(Expr::FunctionCall(fun, new_args), node.ty)
         }
-
-        impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> {
-            fn thir(&self) -> &'a thir::Thir<'tcx> {
-                &self.thir
-            }
-
-            #[instrument(skip(self), level = "debug")]
-            fn visit_expr(&mut self, expr: &thir::Expr<'tcx>) {
-                self.is_poly |= self.expr_is_poly(expr);
-                if !self.is_poly {
-                    visit::walk_expr(self, expr)
-                }
+        &ExprKind::Binary { op, lhs, rhs } if check_binop(op) => {
+            let lhs = recurse_build(tcx, body, lhs, root_span)?;
+            let rhs = recurse_build(tcx, body, rhs, root_span)?;
+            tcx.mk_const(Expr::Binop(op, lhs, rhs), node.ty)
+        }
+        &ExprKind::Unary { op, arg } if check_unop(op) => {
+            let arg = recurse_build(tcx, body, arg, root_span)?;
+            tcx.mk_const(Expr::UnOp(op, arg), node.ty)
+        }
+        // This is necessary so that the following compiles:
+        //
+        // ```
+        // fn foo<const N: usize>(a: [(); N + 1]) {
+        //     bar::<{ N + 1 }>();
+        // }
+        // ```
+        ExprKind::Block { block } => {
+            if let thir::Block { stmts: box [], expr: Some(e), .. } = &body.blocks[*block] {
+                recurse_build(tcx, body, *e, root_span)?
+            } else {
+                maybe_supported_error(GenericConstantTooComplexSub::BlockNotSupported(node.span))?
             }
-
-            #[instrument(skip(self), level = "debug")]
-            fn visit_pat(&mut self, pat: &thir::Pat<'tcx>) {
-                self.is_poly |= self.pat_is_poly(pat);
-                if !self.is_poly {
-                    visit::walk_pat(self, pat);
-                }
+        }
+        // `ExprKind::Use` happens when a `hir::ExprKind::Cast` is a
+        // "coercion cast" i.e. using a coercion or is a no-op.
+        // This is important so that `N as usize as usize` doesnt unify with `N as usize`. (untested)
+        &ExprKind::Use { source } => {
+            let arg = recurse_build(tcx, body, source, root_span)?;
+            tcx.mk_const(Expr::Cast(CastKind::Use, arg, node.ty), node.ty)
+        }
+        &ExprKind::Cast { source } => {
+            let arg = recurse_build(tcx, body, source, root_span)?;
+            tcx.mk_const(Expr::Cast(CastKind::As, arg, node.ty), node.ty)
+        }
+        ExprKind::Borrow { arg, .. } => {
+            let arg_node = &body.exprs[*arg];
+
+            // Skip reborrows for now until we allow Deref/Borrow/AddressOf
+            // expressions.
+            // FIXME(generic_const_exprs): Verify/explain why this is sound
+            if let ExprKind::Deref { arg } = arg_node.kind {
+                recurse_build(tcx, body, arg, root_span)?
+            } else {
+                maybe_supported_error(GenericConstantTooComplexSub::BorrowNotSupported(node.span))?
             }
         }
-
-        let mut is_poly_vis = IsThirPolymorphic { is_poly: false, thir: body };
-        visit::walk_expr(&mut is_poly_vis, &body[body_id]);
-        debug!("AbstractConstBuilder: is_poly={}", is_poly_vis.is_poly);
-        if !is_poly_vis.is_poly {
-            return Ok(None);
+        // FIXME(generic_const_exprs): We may want to support these.
+        ExprKind::AddressOf { .. } | ExprKind::Deref { .. } => maybe_supported_error(
+            GenericConstantTooComplexSub::AddressAndDerefNotSupported(node.span),
+        )?,
+        ExprKind::Repeat { .. } | ExprKind::Array { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::ArrayNotSupported(node.span))?
         }
+        ExprKind::NeverToAny { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::NeverToAnyNotSupported(node.span))?
+        }
+        ExprKind::Tuple { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::TupleNotSupported(node.span))?
+        }
+        ExprKind::Index { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::IndexNotSupported(node.span))?
+        }
+        ExprKind::Field { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::FieldNotSupported(node.span))?
+        }
+        ExprKind::ConstBlock { .. } => {
+            maybe_supported_error(GenericConstantTooComplexSub::ConstBlockNotSupported(node.span))?
+        }
+        ExprKind::Adt(_) => {
+            maybe_supported_error(GenericConstantTooComplexSub::AdtNotSupported(node.span))?
+        }
+        // dont know if this is correct
+        ExprKind::Pointer { .. } => {
+            error(GenericConstantTooComplexSub::PointerNotSupported(node.span))?
+        }
+        ExprKind::Yield { .. } => {
+            error(GenericConstantTooComplexSub::YieldNotSupported(node.span))?
+        }
+        ExprKind::Continue { .. } | ExprKind::Break { .. } | ExprKind::Loop { .. } => {
+            error(GenericConstantTooComplexSub::LoopNotSupported(node.span))?
+        }
+        ExprKind::Box { .. } => error(GenericConstantTooComplexSub::BoxNotSupported(node.span))?,
 
-        Ok(Some(builder))
-    }
-
-    /// We do not allow all binary operations in abstract consts, so filter disallowed ones.
-    fn check_binop(op: mir::BinOp) -> bool {
-        use mir::BinOp::*;
-        match op {
-            Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Shl | Shr | Eq | Lt | Le
-            | Ne | Ge | Gt => true,
-            Offset => false,
+        ExprKind::Unary { .. } => unreachable!(),
+        // we handle valid unary/binary ops above
+        ExprKind::Binary { .. } => {
+            error(GenericConstantTooComplexSub::BinaryNotSupported(node.span))?
+        }
+        ExprKind::LogicalOp { .. } => {
+            error(GenericConstantTooComplexSub::LogicalOpNotSupported(node.span))?
+        }
+        ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+            error(GenericConstantTooComplexSub::AssignNotSupported(node.span))?
+        }
+        ExprKind::Closure { .. } | ExprKind::Return { .. } => {
+            error(GenericConstantTooComplexSub::ClosureAndReturnNotSupported(node.span))?
+        }
+        // let expressions imply control flow
+        ExprKind::Match { .. } | ExprKind::If { .. } | ExprKind::Let { .. } => {
+            error(GenericConstantTooComplexSub::ControlFlowNotSupported(node.span))?
+        }
+        ExprKind::InlineAsm { .. } => {
+            error(GenericConstantTooComplexSub::InlineAsmNotSupported(node.span))?
         }
-    }
 
-    /// While we currently allow all unary operations, we still want to explicitly guard against
-    /// future changes here.
-    fn check_unop(op: mir::UnOp) -> bool {
-        use mir::UnOp::*;
-        match op {
-            Not | Neg => true,
+        // we dont permit let stmts so `VarRef` and `UpvarRef` cant happen
+        ExprKind::VarRef { .. }
+        | ExprKind::UpvarRef { .. }
+        | ExprKind::StaticRef { .. }
+        | ExprKind::ThreadLocalRef(_) => {
+            error(GenericConstantTooComplexSub::OperationNotSupported(node.span))?
         }
-    }
+    })
+}
 
-    /// Builds the abstract const by walking the thir and bailing out when
-    /// encountering an unsupported operation.
-    pub fn build(mut self) -> Result<&'tcx [Node<'tcx>], ErrorGuaranteed> {
-        debug!("AbstractConstBuilder::build: body={:?}", &*self.body);
-        self.recurse_build(self.body_id)?;
+struct IsThirPolymorphic<'a, 'tcx> {
+    is_poly: bool,
+    thir: &'a thir::Thir<'tcx>,
+}
 
-        Ok(self.tcx.arena.alloc_from_iter(self.nodes.into_iter()))
-    }
+fn error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    sub: GenericConstantTooComplexSub,
+    root_span: Span,
+) -> Result<!, ErrorGuaranteed> {
+    let reported = tcx.sess.emit_err(GenericConstantTooComplex {
+        span: root_span,
+        maybe_supported: None,
+        sub,
+    });
+
+    Err(reported)
+}
 
-    fn recurse_build(&mut self, node: thir::ExprId) -> Result<NodeId, ErrorGuaranteed> {
-        use thir::ExprKind;
-        let node = &self.body.exprs[node];
-        Ok(match &node.kind {
-            // I dont know if handling of these 3 is correct
-            &ExprKind::Scope { value, .. } => self.recurse_build(value)?,
-            &ExprKind::PlaceTypeAscription { source, .. }
-            | &ExprKind::ValueTypeAscription { source, .. } => self.recurse_build(source)?,
-            &ExprKind::Literal { lit, neg } => {
-                let sp = node.span;
-                let constant = match self.tcx.at(sp).lit_to_const(LitToConstInput {
-                    lit: &lit.node,
-                    ty: node.ty,
-                    neg,
-                }) {
-                    Ok(c) => c,
-                    Err(LitToConstError::Reported) => self.tcx.const_error(node.ty),
-                    Err(LitToConstError::TypeError) => {
-                        bug!("encountered type error in lit_to_const")
-                    }
-                };
-
-                self.nodes.push(Node::Leaf(constant))
-            }
-            &ExprKind::NonHirLiteral { lit, user_ty: _ } => {
-                let val = ty::ValTree::from_scalar_int(lit);
-                self.nodes.push(Node::Leaf(ty::Const::from_value(self.tcx, val, node.ty)))
-            }
-            &ExprKind::ZstLiteral { user_ty: _ } => {
-                let val = ty::ValTree::zst();
-                self.nodes.push(Node::Leaf(ty::Const::from_value(self.tcx, val, node.ty)))
-            }
-            &ExprKind::NamedConst { def_id, substs, user_ty: _ } => {
-                let uneval =
-                    ty::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
+fn maybe_supported_error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    sub: GenericConstantTooComplexSub,
+    root_span: Span,
+) -> Result<!, ErrorGuaranteed> {
+    let reported = tcx.sess.emit_err(GenericConstantTooComplex {
+        span: root_span,
+        maybe_supported: Some(()),
+        sub,
+    });
+
+    Err(reported)
+}
 
-                let constant = self
-                    .tcx
-                    .mk_const(ty::ConstS { kind: ty::ConstKind::Unevaluated(uneval), ty: node.ty });
+impl<'a, 'tcx> IsThirPolymorphic<'a, 'tcx> {
+    fn expr_is_poly(&mut self, expr: &thir::Expr<'tcx>) -> bool {
+        if expr.ty.has_non_region_param() {
+            return true;
+        }
 
-                self.nodes.push(Node::Leaf(constant))
+        match expr.kind {
+            thir::ExprKind::NamedConst { substs, .. } => substs.has_non_region_param(),
+            thir::ExprKind::ConstParam { .. } => true,
+            thir::ExprKind::Repeat { value, count } => {
+                self.visit_expr(&self.thir()[value]);
+                count.has_non_region_param()
             }
+            _ => false,
+        }
+    }
+    fn pat_is_poly(&mut self, pat: &thir::Pat<'tcx>) -> bool {
+        if pat.ty.has_non_region_param() {
+            return true;
+        }
 
-            ExprKind::ConstParam { param, .. } => {
-                let const_param = self
-                    .tcx
-                    .mk_const(ty::ConstS { kind: ty::ConstKind::Param(*param), ty: node.ty });
-                self.nodes.push(Node::Leaf(const_param))
+        match pat.kind {
+            thir::PatKind::Constant { value } => value.has_non_region_param(),
+            thir::PatKind::Range(box thir::PatRange { lo, hi, .. }) => {
+                lo.has_non_region_param() || hi.has_non_region_param()
             }
+            _ => false,
+        }
+    }
+}
 
-            ExprKind::Call { fun, args, .. } => {
-                let fun = self.recurse_build(*fun)?;
-
-                let mut new_args = Vec::<NodeId>::with_capacity(args.len());
-                for &id in args.iter() {
-                    new_args.push(self.recurse_build(id)?);
-                }
-                let new_args = self.tcx.arena.alloc_slice(&new_args);
-                self.nodes.push(Node::FunctionCall(fun, new_args))
-            }
-            &ExprKind::Binary { op, lhs, rhs } if Self::check_binop(op) => {
-                let lhs = self.recurse_build(lhs)?;
-                let rhs = self.recurse_build(rhs)?;
-                self.nodes.push(Node::Binop(op, lhs, rhs))
-            }
-            &ExprKind::Unary { op, arg } if Self::check_unop(op) => {
-                let arg = self.recurse_build(arg)?;
-                self.nodes.push(Node::UnaryOp(op, arg))
-            }
-            // This is necessary so that the following compiles:
-            //
-            // ```
-            // fn foo<const N: usize>(a: [(); N + 1]) {
-            //     bar::<{ N + 1 }>();
-            // }
-            // ```
-            ExprKind::Block { block } => {
-                if let thir::Block { stmts: box [], expr: Some(e), .. } = &self.body.blocks[*block]
-                {
-                    self.recurse_build(*e)?
-                } else {
-                    self.maybe_supported_error(GenericConstantTooComplexSub::BlockNotSupported(
-                        node.span,
-                    ))?
-                }
-            }
-            // `ExprKind::Use` happens when a `hir::ExprKind::Cast` is a
-            // "coercion cast" i.e. using a coercion or is a no-op.
-            // This is important so that `N as usize as usize` doesnt unify with `N as usize`. (untested)
-            &ExprKind::Use { source } => {
-                let arg = self.recurse_build(source)?;
-                self.nodes.push(Node::Cast(CastKind::Use, arg, node.ty))
-            }
-            &ExprKind::Cast { source } => {
-                let arg = self.recurse_build(source)?;
-                self.nodes.push(Node::Cast(CastKind::As, arg, node.ty))
-            }
-            ExprKind::Borrow { arg, .. } => {
-                let arg_node = &self.body.exprs[*arg];
-
-                // Skip reborrows for now until we allow Deref/Borrow/AddressOf
-                // expressions.
-                // FIXME(generic_const_exprs): Verify/explain why this is sound
-                if let ExprKind::Deref { arg } = arg_node.kind {
-                    self.recurse_build(arg)?
-                } else {
-                    self.maybe_supported_error(GenericConstantTooComplexSub::BorrowNotSupported(
-                        node.span,
-                    ))?
-                }
-            }
-            // FIXME(generic_const_exprs): We may want to support these.
-            ExprKind::AddressOf { .. } | ExprKind::Deref { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::AddressAndDerefNotSupported(node.span),
-            )?,
-            ExprKind::Repeat { .. } | ExprKind::Array { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::ArrayNotSupported(node.span),
-            )?,
-            ExprKind::NeverToAny { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::NeverToAnyNotSupported(node.span),
-            )?,
-            ExprKind::Tuple { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::TupleNotSupported(node.span),
-            )?,
-            ExprKind::Index { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::IndexNotSupported(node.span),
-            )?,
-            ExprKind::Field { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::FieldNotSupported(node.span),
-            )?,
-            ExprKind::ConstBlock { .. } => self.maybe_supported_error(
-                GenericConstantTooComplexSub::ConstBlockNotSupported(node.span),
-            )?,
-            ExprKind::Adt(_) => self
-                .maybe_supported_error(GenericConstantTooComplexSub::AdtNotSupported(node.span))?,
-            // dont know if this is correct
-            ExprKind::Pointer { .. } => {
-                self.error(GenericConstantTooComplexSub::PointerNotSupported(node.span))?
-            }
-            ExprKind::Yield { .. } => {
-                self.error(GenericConstantTooComplexSub::YieldNotSupported(node.span))?
-            }
-            ExprKind::Continue { .. } | ExprKind::Break { .. } | ExprKind::Loop { .. } => {
-                self.error(GenericConstantTooComplexSub::LoopNotSupported(node.span))?
-            }
-            ExprKind::Box { .. } => {
-                self.error(GenericConstantTooComplexSub::BoxNotSupported(node.span))?
-            }
+impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> {
+    fn thir(&self) -> &'a thir::Thir<'tcx> {
+        &self.thir
+    }
 
-            ExprKind::Unary { .. } => unreachable!(),
-            // we handle valid unary/binary ops above
-            ExprKind::Binary { .. } => {
-                self.error(GenericConstantTooComplexSub::BinaryNotSupported(node.span))?
-            }
-            ExprKind::LogicalOp { .. } => {
-                self.error(GenericConstantTooComplexSub::LogicalOpNotSupported(node.span))?
-            }
-            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
-                self.error(GenericConstantTooComplexSub::AssignNotSupported(node.span))?
-            }
-            ExprKind::Closure { .. } | ExprKind::Return { .. } => {
-                self.error(GenericConstantTooComplexSub::ClosureAndReturnNotSupported(node.span))?
-            }
-            // let expressions imply control flow
-            ExprKind::Match { .. } | ExprKind::If { .. } | ExprKind::Let { .. } => {
-                self.error(GenericConstantTooComplexSub::ControlFlowNotSupported(node.span))?
-            }
-            ExprKind::InlineAsm { .. } => {
-                self.error(GenericConstantTooComplexSub::InlineAsmNotSupported(node.span))?
-            }
+    #[instrument(skip(self), level = "debug")]
+    fn visit_expr(&mut self, expr: &thir::Expr<'tcx>) {
+        self.is_poly |= self.expr_is_poly(expr);
+        if !self.is_poly {
+            visit::walk_expr(self, expr)
+        }
+    }
 
-            // we dont permit let stmts so `VarRef` and `UpvarRef` cant happen
-            ExprKind::VarRef { .. }
-            | ExprKind::UpvarRef { .. }
-            | ExprKind::StaticRef { .. }
-            | ExprKind::ThreadLocalRef(_) => {
-                self.error(GenericConstantTooComplexSub::OperationNotSupported(node.span))?
-            }
-        })
+    #[instrument(skip(self), level = "debug")]
+    fn visit_pat(&mut self, pat: &thir::Pat<'tcx>) {
+        self.is_poly |= self.pat_is_poly(pat);
+        if !self.is_poly {
+            visit::walk_pat(self, pat);
+        }
     }
 }
 
@@ -411,7 +352,7 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> {
 pub fn thir_abstract_const<'tcx>(
     tcx: TyCtxt<'tcx>,
     def: ty::WithOptConstParam<LocalDefId>,
-) -> Result<Option<&'tcx [Node<'tcx>]>, ErrorGuaranteed> {
+) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
     if tcx.features().generic_const_exprs {
         match tcx.def_kind(def.did) {
             // FIXME(generic_const_exprs): We currently only do this for anonymous constants,
@@ -424,10 +365,17 @@ pub fn thir_abstract_const<'tcx>(
         }
 
         let body = tcx.thir_body(def)?;
+        let (body, body_id) = (&*body.0.borrow(), body.1);
+
+        let mut is_poly_vis = IsThirPolymorphic { is_poly: false, thir: body };
+        visit::walk_expr(&mut is_poly_vis, &body[body_id]);
+        if !is_poly_vis.is_poly {
+            return Ok(None);
+        }
+
+        let root_span = body.exprs[body_id].span;
 
-        AbstractConstBuilder::new(tcx, (&*body.0.borrow(), body.1))?
-            .map(AbstractConstBuilder::build)
-            .transpose()
+        Some(recurse_build(tcx, body, body_id, root_span)).transpose()
     } else {
         Ok(None)
     }
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 6436713b3..c6f2b16ca 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -202,8 +202,14 @@ fn resolve_associated_item<'tcx>(
             )),
             substs: generator_data.substs,
         }),
+        traits::ImplSource::Future(future_data) => Some(Instance {
+            def: ty::InstanceDef::Item(ty::WithOptConstParam::unknown(
+                future_data.generator_def_id,
+            )),
+            substs: future_data.substs,
+        }),
         traits::ImplSource::Closure(closure_data) => {
-            let trait_closure_kind = tcx.fn_trait_kind_from_lang_item(trait_id).unwrap();
+            let trait_closure_kind = tcx.fn_trait_kind_from_def_id(trait_id).unwrap();
             Instance::resolve_closure(
                 tcx,
                 closure_data.closure_def_id,
@@ -264,8 +270,6 @@ fn resolve_associated_item<'tcx>(
         traits::ImplSource::AutoImpl(..)
         | traits::ImplSource::Param(..)
         | traits::ImplSource::TraitAlias(..)
-        | traits::ImplSource::DiscriminantKind(..)
-        | traits::ImplSource::Pointee(..)
         | traits::ImplSource::TraitUpcasting(_)
         | traits::ImplSource::ConstDestruct(_) => None,
     })
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index 52ba0eee9..fbc055b5d 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -13,13 +13,8 @@ use rustc_span::symbol::Symbol;
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::*;
 
-use std::cmp::{self, Ordering};
+use std::fmt::Debug;
 use std::iter;
-use std::num::NonZeroUsize;
-use std::ops::Bound;
-
-use rand::{seq::SliceRandom, SeedableRng};
-use rand_xoshiro::Xoshiro128StarStar;
 
 use crate::layout_sanity_check::sanity_check_layout;
 
@@ -66,16 +61,6 @@ fn layout_of<'tcx>(
     Ok(layout)
 }
 
-#[derive(Copy, Clone, Debug)]
-enum StructKind {
-    /// A tuple, closure, or univariant which cannot be coerced to unsized.
-    AlwaysSized,
-    /// A univariant, the last field of which may be coerced to unsized.
-    MaybeUnsized,
-    /// A univariant, but with a prefix of an arbitrary size & alignment (e.g., enum tag).
-    Prefixed(Size, Align),
-}
-
 // Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
 // This is used to go between `memory_index` (source field order to memory order)
 // and `inverse_memory_index` (memory order to source field order).
@@ -89,40 +74,13 @@ fn invert_mapping(map: &[u32]) -> Vec<u32> {
     inverse
 }
 
-fn scalar_pair<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, a: Scalar, b: Scalar) -> LayoutS<'tcx> {
-    let dl = cx.data_layout();
-    let b_align = b.align(dl);
-    let align = a.align(dl).max(b_align).max(dl.aggregate_align);
-    let b_offset = a.size(dl).align_to(b_align.abi);
-    let size = (b_offset + b.size(dl)).align_to(align.abi);
-
-    // HACK(nox): We iter on `b` and then `a` because `max_by_key`
-    // returns the last maximum.
-    let largest_niche = Niche::from_scalar(dl, b_offset, b)
-        .into_iter()
-        .chain(Niche::from_scalar(dl, Size::ZERO, a))
-        .max_by_key(|niche| niche.available(dl));
-
-    LayoutS {
-        variants: Variants::Single { index: VariantIdx::new(0) },
-        fields: FieldsShape::Arbitrary {
-            offsets: vec![Size::ZERO, b_offset],
-            memory_index: vec![0, 1],
-        },
-        abi: Abi::ScalarPair(a, b),
-        largest_niche,
-        align,
-        size,
-    }
-}
-
 fn univariant_uninterned<'tcx>(
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
     ty: Ty<'tcx>,
     fields: &[TyAndLayout<'_>],
     repr: &ReprOptions,
     kind: StructKind,
-) -> Result<LayoutS<'tcx>, LayoutError<'tcx>> {
+) -> Result<LayoutS<VariantIdx>, LayoutError<'tcx>> {
     let dl = cx.data_layout();
     let pack = repr.pack;
     if pack.is_some() && repr.align.is_some() {
@@ -130,208 +88,7 @@ fn univariant_uninterned<'tcx>(
         return Err(LayoutError::Unknown(ty));
     }
 
-    let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
-
-    let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
-
-    let optimize = !repr.inhibit_struct_field_reordering_opt();
-    if optimize {
-        let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
-        let optimizing = &mut inverse_memory_index[..end];
-        let field_align = |f: &TyAndLayout<'_>| {
-            if let Some(pack) = pack { f.align.abi.min(pack) } else { f.align.abi }
-        };
-
-        // If `-Z randomize-layout` was enabled for the type definition we can shuffle
-        // the field ordering to try and catch some code making assumptions about layouts
-        // we don't guarantee
-        if repr.can_randomize_type_layout() {
-            // `ReprOptions.layout_seed` is a deterministic seed that we can use to
-            // randomize field ordering with
-            let mut rng = Xoshiro128StarStar::seed_from_u64(repr.field_shuffle_seed);
-
-            // Shuffle the ordering of the fields
-            optimizing.shuffle(&mut rng);
-
-            // Otherwise we just leave things alone and actually optimize the type's fields
-        } else {
-            match kind {
-                StructKind::AlwaysSized | StructKind::MaybeUnsized => {
-                    optimizing.sort_by_key(|&x| {
-                        // Place ZSTs first to avoid "interesting offsets",
-                        // especially with only one or two non-ZST fields.
-                        let f = &fields[x as usize];
-                        (!f.is_zst(), cmp::Reverse(field_align(f)))
-                    });
-                }
-
-                StructKind::Prefixed(..) => {
-                    // Sort in ascending alignment so that the layout stays optimal
-                    // regardless of the prefix
-                    optimizing.sort_by_key(|&x| field_align(&fields[x as usize]));
-                }
-            }
-
-            // FIXME(Kixiron): We can always shuffle fields within a given alignment class
-            //                 regardless of the status of `-Z randomize-layout`
-        }
-    }
-
-    // inverse_memory_index holds field indices by increasing memory offset.
-    // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
-    // We now write field offsets to the corresponding offset slot;
-    // field 5 with offset 0 puts 0 in offsets[5].
-    // At the bottom of this function, we invert `inverse_memory_index` to
-    // produce `memory_index` (see `invert_mapping`).
-
-    let mut sized = true;
-    let mut offsets = vec![Size::ZERO; fields.len()];
-    let mut offset = Size::ZERO;
-    let mut largest_niche = None;
-    let mut largest_niche_available = 0;
-
-    if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
-        let prefix_align =
-            if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
-        align = align.max(AbiAndPrefAlign::new(prefix_align));
-        offset = prefix_size.align_to(prefix_align);
-    }
-
-    for &i in &inverse_memory_index {
-        let field = fields[i as usize];
-        if !sized {
-            cx.tcx.sess.delay_span_bug(
-                DUMMY_SP,
-                &format!(
-                    "univariant: field #{} of `{}` comes after unsized field",
-                    offsets.len(),
-                    ty
-                ),
-            );
-        }
-
-        if field.is_unsized() {
-            sized = false;
-        }
-
-        // Invariant: offset < dl.obj_size_bound() <= 1<<61
-        let field_align = if let Some(pack) = pack {
-            field.align.min(AbiAndPrefAlign::new(pack))
-        } else {
-            field.align
-        };
-        offset = offset.align_to(field_align.abi);
-        align = align.max(field_align);
-
-        debug!("univariant offset: {:?} field: {:#?}", offset, field);
-        offsets[i as usize] = offset;
-
-        if let Some(mut niche) = field.largest_niche {
-            let available = niche.available(dl);
-            if available > largest_niche_available {
-                largest_niche_available = available;
-                niche.offset += offset;
-                largest_niche = Some(niche);
-            }
-        }
-
-        offset = offset.checked_add(field.size, dl).ok_or(LayoutError::SizeOverflow(ty))?;
-    }
-
-    if let Some(repr_align) = repr.align {
-        align = align.max(AbiAndPrefAlign::new(repr_align));
-    }
-
-    debug!("univariant min_size: {:?}", offset);
-    let min_size = offset;
-
-    // As stated above, inverse_memory_index holds field indices by increasing offset.
-    // This makes it an already-sorted view of the offsets vec.
-    // To invert it, consider:
-    // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
-    // Field 5 would be the first element, so memory_index is i:
-    // Note: if we didn't optimize, it's already right.
-
-    let memory_index =
-        if optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index };
-
-    let size = min_size.align_to(align.abi);
-    let mut abi = Abi::Aggregate { sized };
-
-    // Unpack newtype ABIs and find scalar pairs.
-    if sized && size.bytes() > 0 {
-        // All other fields must be ZSTs.
-        let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.is_zst());
-
-        match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
-            // We have exactly one non-ZST field.
-            (Some((i, field)), None, None) => {
-                // Field fills the struct and it has a scalar or scalar pair ABI.
-                if offsets[i].bytes() == 0 && align.abi == field.align.abi && size == field.size {
-                    match field.abi {
-                        // For plain scalars, or vectors of them, we can't unpack
-                        // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
-                            abi = field.abi;
-                        }
-                        // But scalar pairs are Rust-specific and get
-                        // treated as aggregates by C ABIs anyway.
-                        Abi::ScalarPair(..) => {
-                            abi = field.abi;
-                        }
-                        _ => {}
-                    }
-                }
-            }
-
-            // Two non-ZST fields, and they're both scalars.
-            (Some((i, a)), Some((j, b)), None) => {
-                match (a.abi, b.abi) {
-                    (Abi::Scalar(a), Abi::Scalar(b)) => {
-                        // Order by the memory placement, not source order.
-                        let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
-                            ((i, a), (j, b))
-                        } else {
-                            ((j, b), (i, a))
-                        };
-                        let pair = scalar_pair(cx, a, b);
-                        let pair_offsets = match pair.fields {
-                            FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
-                                assert_eq!(memory_index, &[0, 1]);
-                                offsets
-                            }
-                            _ => bug!(),
-                        };
-                        if offsets[i] == pair_offsets[0]
-                            && offsets[j] == pair_offsets[1]
-                            && align == pair.align
-                            && size == pair.size
-                        {
-                            // We can use `ScalarPair` only when it matches our
-                            // already computed layout (including `#[repr(C)]`).
-                            abi = pair.abi;
-                        }
-                    }
-                    _ => {}
-                }
-            }
-
-            _ => {}
-        }
-    }
-
-    if fields.iter().any(|f| f.abi.is_uninhabited()) {
-        abi = Abi::Uninhabited;
-    }
-
-    Ok(LayoutS {
-        variants: Variants::Single { index: VariantIdx::new(0) },
-        fields: FieldsShape::Arbitrary { offsets, memory_index },
-        abi,
-        largest_niche,
-        align,
-        size,
-    })
+    cx.univariant(dl, fields, repr, kind).ok_or(LayoutError::SizeOverflow(ty))
 }
 
 fn layout_of_uncached<'tcx>(
@@ -382,14 +139,7 @@ fn layout_of_uncached<'tcx>(
         }
 
         // The never type.
-        ty::Never => tcx.intern_layout(LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
-            fields: FieldsShape::Primitive,
-            abi: Abi::Uninhabited,
-            largest_niche: None,
-            align: dl.i8_align,
-            size: Size::ZERO,
-        }),
+        ty::Never => tcx.intern_layout(cx.layout_of_never_type()),
 
         // Potentially-wide pointers.
         ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
@@ -418,7 +168,7 @@ fn layout_of_uncached<'tcx>(
             };
 
             // Effectively a (ptr, meta) tuple.
-            tcx.intern_layout(scalar_pair(cx, data_ptr, metadata))
+            tcx.intern_layout(cx.scalar_pair(data_ptr, metadata))
         }
 
         ty::Dynamic(_, _, ty::DynStar) => {
@@ -426,7 +176,7 @@ fn layout_of_uncached<'tcx>(
             data.valid_range_mut().start = 0;
             let mut vtable = scalar_unit(Pointer);
             vtable.valid_range_mut().start = 1;
-            tcx.intern_layout(scalar_pair(cx, data, vtable))
+            tcx.intern_layout(cx.scalar_pair(data, vtable))
         }
 
         // Arrays and slices.
@@ -442,8 +192,7 @@ fn layout_of_uncached<'tcx>(
             let element = cx.layout_of(element)?;
             let size = element.size.checked_mul(count, dl).ok_or(LayoutError::SizeOverflow(ty))?;
 
-            let abi = if count != 0 && tcx.conservative_is_privately_uninhabited(param_env.and(ty))
-            {
+            let abi = if count != 0 && ty.is_privately_uninhabited(tcx, param_env) {
                 Abi::Uninhabited
             } else {
                 Abi::Aggregate { sized: true }
@@ -576,8 +325,8 @@ fn layout_of_uncached<'tcx>(
 
                 // Extract the number of elements from the layout of the array field:
                 let FieldsShape::Array { count, .. } = cx.layout_of(f0_ty)?.layout.fields() else {
-                        return Err(LayoutError::Unknown(ty));
-                    };
+                    return Err(LayoutError::Unknown(ty));
+                };
 
                 (*e_ty, *count, true)
             } else {
@@ -602,14 +351,14 @@ fn layout_of_uncached<'tcx>(
             // Compute the ABI of the element type:
             let e_ly = cx.layout_of(e_ty)?;
             let Abi::Scalar(e_abi) = e_ly.abi else {
-                    // This error isn't caught in typeck, e.g., if
-                    // the element type of the vector is generic.
-                    tcx.sess.fatal(&format!(
-                        "monomorphising SIMD type `{}` with a non-primitive-scalar \
-                        (integer/float/pointer) element type `{}`",
-                        ty, e_ty
-                    ))
-                };
+                // This error isn't caught in typeck, e.g., if
+                // the element type of the vector is generic.
+                tcx.sess.fatal(&format!(
+                    "monomorphising SIMD type `{}` with a non-primitive-scalar \
+                    (integer/float/pointer) element type `{}`",
+                    ty, e_ty
+                ))
+            };
 
             // Compute the size and alignment of the vector:
             let size = e_ly.size.checked_mul(e_len, dl).ok_or(LayoutError::SizeOverflow(ty))?;
@@ -656,681 +405,41 @@ fn layout_of_uncached<'tcx>(
                     return Err(LayoutError::Unknown(ty));
                 }
 
-                let mut align =
-                    if def.repr().pack.is_some() { dl.i8_align } else { dl.aggregate_align };
-
-                if let Some(repr_align) = def.repr().align {
-                    align = align.max(AbiAndPrefAlign::new(repr_align));
-                }
-
-                let optimize = !def.repr().inhibit_union_abi_opt();
-                let mut size = Size::ZERO;
-                let mut abi = Abi::Aggregate { sized: true };
-                let index = VariantIdx::new(0);
-                for field in &variants[index] {
-                    assert!(!field.is_unsized());
-                    align = align.max(field.align);
-
-                    // If all non-ZST fields have the same ABI, forward this ABI
-                    if optimize && !field.is_zst() {
-                        // Discard valid range information and allow undef
-                        let field_abi = match field.abi {
-                            Abi::Scalar(x) => Abi::Scalar(x.to_union()),
-                            Abi::ScalarPair(x, y) => Abi::ScalarPair(x.to_union(), y.to_union()),
-                            Abi::Vector { element: x, count } => {
-                                Abi::Vector { element: x.to_union(), count }
-                            }
-                            Abi::Uninhabited | Abi::Aggregate { .. } => {
-                                Abi::Aggregate { sized: true }
-                            }
-                        };
-
-                        if size == Size::ZERO {
-                            // first non ZST: initialize 'abi'
-                            abi = field_abi;
-                        } else if abi != field_abi {
-                            // different fields have different ABI: reset to Aggregate
-                            abi = Abi::Aggregate { sized: true };
-                        }
-                    }
-
-                    size = cmp::max(size, field.size);
-                }
-
-                if let Some(pack) = def.repr().pack {
-                    align = align.min(AbiAndPrefAlign::new(pack));
-                }
-
-                return Ok(tcx.intern_layout(LayoutS {
-                    variants: Variants::Single { index },
-                    fields: FieldsShape::Union(
-                        NonZeroUsize::new(variants[index].len()).ok_or(LayoutError::Unknown(ty))?,
-                    ),
-                    abi,
-                    largest_niche: None,
-                    align,
-                    size: size.align_to(align.abi),
-                }));
-            }
-
-            // A variant is absent if it's uninhabited and only has ZST fields.
-            // Present uninhabited variants only require space for their fields,
-            // but *not* an encoding of the discriminant (e.g., a tag value).
-            // See issue #49298 for more details on the need to leave space
-            // for non-ZST uninhabited data (mostly partial initialization).
-            let absent = |fields: &[TyAndLayout<'_>]| {
-                let uninhabited = fields.iter().any(|f| f.abi.is_uninhabited());
-                let is_zst = fields.iter().all(|f| f.is_zst());
-                uninhabited && is_zst
-            };
-            let (present_first, present_second) = {
-                let mut present_variants = variants
-                    .iter_enumerated()
-                    .filter_map(|(i, v)| if absent(v) { None } else { Some(i) });
-                (present_variants.next(), present_variants.next())
-            };
-            let present_first = match present_first {
-                Some(present_first) => present_first,
-                // Uninhabited because it has no variants, or only absent ones.
-                None if def.is_enum() => {
-                    return Ok(tcx.layout_of(param_env.and(tcx.types.never))?.layout);
-                }
-                // If it's a struct, still compute a layout so that we can still compute the
-                // field offsets.
-                None => VariantIdx::new(0),
-            };
-
-            let is_struct = !def.is_enum() ||
-                    // Only one variant is present.
-                    (present_second.is_none() &&
-                        // Representation optimizations are allowed.
-                        !def.repr().inhibit_enum_layout_opt());
-            if is_struct {
-                // Struct, or univariant enum equivalent to a struct.
-                // (Typechecking will reject discriminant-sizing attrs.)
-
-                let v = present_first;
-                let kind = if def.is_enum() || variants[v].is_empty() {
-                    StructKind::AlwaysSized
-                } else {
-                    let param_env = tcx.param_env(def.did());
-                    let last_field = def.variant(v).fields.last().unwrap();
-                    let always_sized = tcx.type_of(last_field.did).is_sized(tcx, param_env);
-                    if !always_sized { StructKind::MaybeUnsized } else { StructKind::AlwaysSized }
-                };
-
-                let mut st = univariant_uninterned(cx, ty, &variants[v], &def.repr(), kind)?;
-                st.variants = Variants::Single { index: v };
-
-                if def.is_unsafe_cell() {
-                    let hide_niches = |scalar: &mut _| match scalar {
-                        Scalar::Initialized { value, valid_range } => {
-                            *valid_range = WrappingRange::full(value.size(dl))
-                        }
-                        // Already doesn't have any niches
-                        Scalar::Union { .. } => {}
-                    };
-                    match &mut st.abi {
-                        Abi::Uninhabited => {}
-                        Abi::Scalar(scalar) => hide_niches(scalar),
-                        Abi::ScalarPair(a, b) => {
-                            hide_niches(a);
-                            hide_niches(b);
-                        }
-                        Abi::Vector { element, count: _ } => hide_niches(element),
-                        Abi::Aggregate { sized: _ } => {}
-                    }
-                    st.largest_niche = None;
-                    return Ok(tcx.intern_layout(st));
-                }
-
-                let (start, end) = cx.tcx.layout_scalar_valid_range(def.did());
-                match st.abi {
-                    Abi::Scalar(ref mut scalar) | Abi::ScalarPair(ref mut scalar, _) => {
-                        // the asserts ensure that we are not using the
-                        // `#[rustc_layout_scalar_valid_range(n)]`
-                        // attribute to widen the range of anything as that would probably
-                        // result in UB somewhere
-                        // FIXME(eddyb) the asserts are probably not needed,
-                        // as larger validity ranges would result in missed
-                        // optimizations, *not* wrongly assuming the inner
-                        // value is valid. e.g. unions enlarge validity ranges,
-                        // because the values may be uninitialized.
-                        if let Bound::Included(start) = start {
-                            // FIXME(eddyb) this might be incorrect - it doesn't
-                            // account for wrap-around (end < start) ranges.
-                            let valid_range = scalar.valid_range_mut();
-                            assert!(valid_range.start <= start);
-                            valid_range.start = start;
-                        }
-                        if let Bound::Included(end) = end {
-                            // FIXME(eddyb) this might be incorrect - it doesn't
-                            // account for wrap-around (end < start) ranges.
-                            let valid_range = scalar.valid_range_mut();
-                            assert!(valid_range.end >= end);
-                            valid_range.end = end;
-                        }
-
-                        // Update `largest_niche` if we have introduced a larger niche.
-                        let niche = Niche::from_scalar(dl, Size::ZERO, *scalar);
-                        if let Some(niche) = niche {
-                            match st.largest_niche {
-                                Some(largest_niche) => {
-                                    // Replace the existing niche even if they're equal,
-                                    // because this one is at a lower offset.
-                                    if largest_niche.available(dl) <= niche.available(dl) {
-                                        st.largest_niche = Some(niche);
-                                    }
-                                }
-                                None => st.largest_niche = Some(niche),
-                            }
-                        }
-                    }
-                    _ => assert!(
-                        start == Bound::Unbounded && end == Bound::Unbounded,
-                        "nonscalar layout for layout_scalar_valid_range type {:?}: {:#?}",
-                        def,
-                        st,
-                    ),
-                }
-
-                return Ok(tcx.intern_layout(st));
-            }
-
-            // At this point, we have handled all unions and
-            // structs. (We have also handled univariant enums
-            // that allow representation optimization.)
-            assert!(def.is_enum());
-
-            // Until we've decided whether to use the tagged or
-            // niche filling LayoutS, we don't want to intern the
-            // variant layouts, so we can't store them in the
-            // overall LayoutS. Store the overall LayoutS
-            // and the variant LayoutSs here until then.
-            struct TmpLayout<'tcx> {
-                layout: LayoutS<'tcx>,
-                variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
+                return Ok(tcx.intern_layout(
+                    cx.layout_of_union(&def.repr(), &variants).ok_or(LayoutError::Unknown(ty))?,
+                ));
             }
 
-            let calculate_niche_filling_layout =
-                || -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
-                    // The current code for niche-filling relies on variant indices
-                    // instead of actual discriminants, so enums with
-                    // explicit discriminants (RFC #2363) would misbehave.
-                    if def.repr().inhibit_enum_layout_opt()
+            tcx.intern_layout(
+                cx.layout_of_struct_or_enum(
+                    &def.repr(),
+                    &variants,
+                    def.is_enum(),
+                    def.is_unsafe_cell(),
+                    tcx.layout_scalar_valid_range(def.did()),
+                    |min, max| Integer::repr_discr(tcx, ty, &def.repr(), min, max),
+                    def.is_enum()
+                        .then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val as i128)))
+                        .into_iter()
+                        .flatten(),
+                    def.repr().inhibit_enum_layout_opt()
                         || def
                             .variants()
                             .iter_enumerated()
-                            .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
-                    {
-                        return Ok(None);
-                    }
-
-                    if variants.len() < 2 {
-                        return Ok(None);
-                    }
-
-                    let mut align = dl.aggregate_align;
-                    let mut variant_layouts = variants
-                        .iter_enumerated()
-                        .map(|(j, v)| {
-                            let mut st = univariant_uninterned(
-                                cx,
-                                ty,
-                                v,
-                                &def.repr(),
-                                StructKind::AlwaysSized,
-                            )?;
-                            st.variants = Variants::Single { index: j };
-
-                            align = align.max(st.align);
-
-                            Ok(st)
-                        })
-                        .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
-                    let largest_variant_index = match variant_layouts
-                        .iter_enumerated()
-                        .max_by_key(|(_i, layout)| layout.size.bytes())
-                        .map(|(i, _layout)| i)
-                    {
-                        None => return Ok(None),
-                        Some(i) => i,
-                    };
-
-                    let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
-                    let needs_disc = |index: VariantIdx| {
-                        index != largest_variant_index && !absent(&variants[index])
-                    };
-                    let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
-                        ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
-
-                    let count = niche_variants.size_hint().1.unwrap() as u128;
-
-                    // Find the field with the largest niche
-                    let (field_index, niche, (niche_start, niche_scalar)) = match variants
-                        [largest_variant_index]
-                        .iter()
-                        .enumerate()
-                        .filter_map(|(j, field)| Some((j, field.largest_niche?)))
-                        .max_by_key(|(_, niche)| niche.available(dl))
-                        .and_then(|(j, niche)| Some((j, niche, niche.reserve(cx, count)?)))
+                            .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32())),
                     {
-                        None => return Ok(None),
-                        Some(x) => x,
-                    };
-
-                    let niche_offset = niche.offset
-                        + variant_layouts[largest_variant_index].fields.offset(field_index);
-                    let niche_size = niche.value.size(dl);
-                    let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
-
-                    let all_variants_fit =
-                        variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
-                            if i == largest_variant_index {
-                                return true;
-                            }
-
-                            layout.largest_niche = None;
-
-                            if layout.size <= niche_offset {
-                                // This variant will fit before the niche.
-                                return true;
-                            }
-
-                            // Determine if it'll fit after the niche.
-                            let this_align = layout.align.abi;
-                            let this_offset = (niche_offset + niche_size).align_to(this_align);
-
-                            if this_offset + layout.size > size {
-                                return false;
-                            }
-
-                            // It'll fit, but we need to make some adjustments.
-                            match layout.fields {
-                                FieldsShape::Arbitrary { ref mut offsets, .. } => {
-                                    for (j, offset) in offsets.iter_mut().enumerate() {
-                                        if !variants[i][j].is_zst() {
-                                            *offset += this_offset;
-                                        }
-                                    }
-                                }
-                                _ => {
-                                    panic!("Layout of fields should be Arbitrary for variants")
-                                }
-                            }
-
-                            // It can't be a Scalar or ScalarPair because the offset isn't 0.
-                            if !layout.abi.is_uninhabited() {
-                                layout.abi = Abi::Aggregate { sized: true };
-                            }
-                            layout.size += this_offset;
-
-                            true
-                        });
-
-                    if !all_variants_fit {
-                        return Ok(None);
-                    }
-
-                    let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
-
-                    let others_zst = variant_layouts
-                        .iter_enumerated()
-                        .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO);
-                    let same_size = size == variant_layouts[largest_variant_index].size;
-                    let same_align = align == variant_layouts[largest_variant_index].align;
-
-                    let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
-                        Abi::Uninhabited
-                    } else if same_size && same_align && others_zst {
-                        match variant_layouts[largest_variant_index].abi {
-                            // When the total alignment and size match, we can use the
-                            // same ABI as the scalar variant with the reserved niche.
-                            Abi::Scalar(_) => Abi::Scalar(niche_scalar),
-                            Abi::ScalarPair(first, second) => {
-                                // Only the niche is guaranteed to be initialised,
-                                // so use union layouts for the other primitive.
-                                if niche_offset == Size::ZERO {
-                                    Abi::ScalarPair(niche_scalar, second.to_union())
-                                } else {
-                                    Abi::ScalarPair(first.to_union(), niche_scalar)
-                                }
-                            }
-                            _ => Abi::Aggregate { sized: true },
-                        }
-                    } else {
-                        Abi::Aggregate { sized: true }
-                    };
-
-                    let layout = LayoutS {
-                        variants: Variants::Multiple {
-                            tag: niche_scalar,
-                            tag_encoding: TagEncoding::Niche {
-                                untagged_variant: largest_variant_index,
-                                niche_variants,
-                                niche_start,
-                            },
-                            tag_field: 0,
-                            variants: IndexVec::new(),
-                        },
-                        fields: FieldsShape::Arbitrary {
-                            offsets: vec![niche_offset],
-                            memory_index: vec![0],
-                        },
-                        abi,
-                        largest_niche,
-                        size,
-                        align,
-                    };
-
-                    Ok(Some(TmpLayout { layout, variants: variant_layouts }))
-                };
-
-            let niche_filling_layout = calculate_niche_filling_layout()?;
-
-            let (mut min, mut max) = (i128::MAX, i128::MIN);
-            let discr_type = def.repr().discr_type();
-            let bits = Integer::from_attr(cx, discr_type).size().bits();
-            for (i, discr) in def.discriminants(tcx) {
-                if variants[i].iter().any(|f| f.abi.is_uninhabited()) {
-                    continue;
-                }
-                let mut x = discr.val as i128;
-                if discr_type.is_signed() {
-                    // sign extend the raw representation to be an i128
-                    x = (x << (128 - bits)) >> (128 - bits);
-                }
-                if x < min {
-                    min = x;
-                }
-                if x > max {
-                    max = x;
-                }
-            }
-            // We might have no inhabited variants, so pretend there's at least one.
-            if (min, max) == (i128::MAX, i128::MIN) {
-                min = 0;
-                max = 0;
-            }
-            assert!(min <= max, "discriminant range is {}...{}", min, max);
-            let (min_ity, signed) = Integer::repr_discr(tcx, ty, &def.repr(), min, max);
-
-            let mut align = dl.aggregate_align;
-            let mut size = Size::ZERO;
-
-            // We're interested in the smallest alignment, so start large.
-            let mut start_align = Align::from_bytes(256).unwrap();
-            assert_eq!(Integer::for_align(dl, start_align), None);
-
-            // repr(C) on an enum tells us to make a (tag, union) layout,
-            // so we need to grow the prefix alignment to be at least
-            // the alignment of the union. (This value is used both for
-            // determining the alignment of the overall enum, and the
-            // determining the alignment of the payload after the tag.)
-            let mut prefix_align = min_ity.align(dl).abi;
-            if def.repr().c() {
-                for fields in &variants {
-                    for field in fields {
-                        prefix_align = prefix_align.max(field.align.abi);
-                    }
-                }
-            }
-
-            // Create the set of structs that represent each variant.
-            let mut layout_variants = variants
-                .iter_enumerated()
-                .map(|(i, field_layouts)| {
-                    let mut st = univariant_uninterned(
-                        cx,
-                        ty,
-                        &field_layouts,
-                        &def.repr(),
-                        StructKind::Prefixed(min_ity.size(), prefix_align),
-                    )?;
-                    st.variants = Variants::Single { index: i };
-                    // Find the first field we can't move later
-                    // to make room for a larger discriminant.
-                    for field in st.fields.index_by_increasing_offset().map(|j| field_layouts[j]) {
-                        if !field.is_zst() || field.align.abi.bytes() != 1 {
-                            start_align = start_align.min(field.align.abi);
-                            break;
-                        }
-                    }
-                    size = cmp::max(size, st.size);
-                    align = align.max(st.align);
-                    Ok(st)
-                })
-                .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
-            // Align the maximum variant size to the largest alignment.
-            size = size.align_to(align.abi);
-
-            if size.bytes() >= dl.obj_size_bound() {
-                return Err(LayoutError::SizeOverflow(ty));
-            }
-
-            let typeck_ity = Integer::from_attr(dl, def.repr().discr_type());
-            if typeck_ity < min_ity {
-                // It is a bug if Layout decided on a greater discriminant size than typeck for
-                // some reason at this point (based on values discriminant can take on). Mostly
-                // because this discriminant will be loaded, and then stored into variable of
-                // type calculated by typeck. Consider such case (a bug): typeck decided on
-                // byte-sized discriminant, but layout thinks we need a 16-bit to store all
-                // discriminant values. That would be a bug, because then, in codegen, in order
-                // to store this 16-bit discriminant into 8-bit sized temporary some of the
-                // space necessary to represent would have to be discarded (or layout is wrong
-                // on thinking it needs 16 bits)
-                bug!(
-                    "layout decided on a larger discriminant type ({:?}) than typeck ({:?})",
-                    min_ity,
-                    typeck_ity
-                );
-                // However, it is fine to make discr type however large (as an optimisation)
-                // after this point – we’ll just truncate the value we load in codegen.
-            }
-
-            // Check to see if we should use a different type for the
-            // discriminant. We can safely use a type with the same size
-            // as the alignment of the first field of each variant.
-            // We increase the size of the discriminant to avoid LLVM copying
-            // padding when it doesn't need to. This normally causes unaligned
-            // load/stores and excessive memcpy/memset operations. By using a
-            // bigger integer size, LLVM can be sure about its contents and
-            // won't be so conservative.
-
-            // Use the initial field alignment
-            let mut ity = if def.repr().c() || def.repr().int.is_some() {
-                min_ity
-            } else {
-                Integer::for_align(dl, start_align).unwrap_or(min_ity)
-            };
-
-            // If the alignment is not larger than the chosen discriminant size,
-            // don't use the alignment as the final size.
-            if ity <= min_ity {
-                ity = min_ity;
-            } else {
-                // Patch up the variants' first few fields.
-                let old_ity_size = min_ity.size();
-                let new_ity_size = ity.size();
-                for variant in &mut layout_variants {
-                    match variant.fields {
-                        FieldsShape::Arbitrary { ref mut offsets, .. } => {
-                            for i in offsets {
-                                if *i <= old_ity_size {
-                                    assert_eq!(*i, old_ity_size);
-                                    *i = new_ity_size;
+                        let param_env = tcx.param_env(def.did());
+                        def.is_struct()
+                            && match def.variants().iter().next().and_then(|x| x.fields.last()) {
+                                Some(last_field) => {
+                                    tcx.type_of(last_field.did).is_sized(tcx, param_env)
                                 }
+                                None => false,
                             }
-                            // We might be making the struct larger.
-                            if variant.size <= old_ity_size {
-                                variant.size = new_ity_size;
-                            }
-                        }
-                        _ => bug!(),
-                    }
-                }
-            }
-
-            let tag_mask = ity.size().unsigned_int_max();
-            let tag = Scalar::Initialized {
-                value: Int(ity, signed),
-                valid_range: WrappingRange {
-                    start: (min as u128 & tag_mask),
-                    end: (max as u128 & tag_mask),
-                },
-            };
-            let mut abi = Abi::Aggregate { sized: true };
-
-            if layout_variants.iter().all(|v| v.abi.is_uninhabited()) {
-                abi = Abi::Uninhabited;
-            } else if tag.size(dl) == size {
-                // Make sure we only use scalar layout when the enum is entirely its
-                // own tag (i.e. it has no padding nor any non-ZST variant fields).
-                abi = Abi::Scalar(tag);
-            } else {
-                // Try to use a ScalarPair for all tagged enums.
-                let mut common_prim = None;
-                let mut common_prim_initialized_in_all_variants = true;
-                for (field_layouts, layout_variant) in iter::zip(&variants, &layout_variants) {
-                    let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
-                            bug!();
-                        };
-                    let mut fields = iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
-                    let (field, offset) = match (fields.next(), fields.next()) {
-                        (None, None) => {
-                            common_prim_initialized_in_all_variants = false;
-                            continue;
-                        }
-                        (Some(pair), None) => pair,
-                        _ => {
-                            common_prim = None;
-                            break;
-                        }
-                    };
-                    let prim = match field.abi {
-                        Abi::Scalar(scalar) => {
-                            common_prim_initialized_in_all_variants &=
-                                matches!(scalar, Scalar::Initialized { .. });
-                            scalar.primitive()
-                        }
-                        _ => {
-                            common_prim = None;
-                            break;
-                        }
-                    };
-                    if let Some(pair) = common_prim {
-                        // This is pretty conservative. We could go fancier
-                        // by conflating things like i32 and u32, or even
-                        // realising that (u8, u8) could just cohabit with
-                        // u16 or even u32.
-                        if pair != (prim, offset) {
-                            common_prim = None;
-                            break;
-                        }
-                    } else {
-                        common_prim = Some((prim, offset));
-                    }
-                }
-                if let Some((prim, offset)) = common_prim {
-                    let prim_scalar = if common_prim_initialized_in_all_variants {
-                        scalar_unit(prim)
-                    } else {
-                        // Common prim might be uninit.
-                        Scalar::Union { value: prim }
-                    };
-                    let pair = scalar_pair(cx, tag, prim_scalar);
-                    let pair_offsets = match pair.fields {
-                        FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
-                            assert_eq!(memory_index, &[0, 1]);
-                            offsets
-                        }
-                        _ => bug!(),
-                    };
-                    if pair_offsets[0] == Size::ZERO
-                        && pair_offsets[1] == *offset
-                        && align == pair.align
-                        && size == pair.size
-                    {
-                        // We can use `ScalarPair` only when it matches our
-                        // already computed layout (including `#[repr(C)]`).
-                        abi = pair.abi;
-                    }
-                }
-            }
-
-            // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the
-            // variants to ensure they are consistent. This is because a downcast is
-            // semantically a NOP, and thus should not affect layout.
-            if matches!(abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
-                for variant in &mut layout_variants {
-                    // We only do this for variants with fields; the others are not accessed anyway.
-                    // Also do not overwrite any already existing "clever" ABIs.
-                    if variant.fields.count() > 0 && matches!(variant.abi, Abi::Aggregate { .. }) {
-                        variant.abi = abi;
-                        // Also need to bump up the size and alignment, so that the entire value fits in here.
-                        variant.size = cmp::max(variant.size, size);
-                        variant.align.abi = cmp::max(variant.align.abi, align.abi);
-                    }
-                }
-            }
-
-            let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
-
-            let tagged_layout = LayoutS {
-                variants: Variants::Multiple {
-                    tag,
-                    tag_encoding: TagEncoding::Direct,
-                    tag_field: 0,
-                    variants: IndexVec::new(),
-                },
-                fields: FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] },
-                largest_niche,
-                abi,
-                align,
-                size,
-            };
-
-            let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
-
-            let mut best_layout = match (tagged_layout, niche_filling_layout) {
-                (tl, Some(nl)) => {
-                    // Pick the smaller layout; otherwise,
-                    // pick the layout with the larger niche; otherwise,
-                    // pick tagged as it has simpler codegen.
-                    use Ordering::*;
-                    let niche_size = |tmp_l: &TmpLayout<'_>| {
-                        tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
-                    };
-                    match (
-                        tl.layout.size.cmp(&nl.layout.size),
-                        niche_size(&tl).cmp(&niche_size(&nl)),
-                    ) {
-                        (Greater, _) => nl,
-                        (Equal, Less) => nl,
-                        _ => tl,
-                    }
-                }
-                (tl, None) => tl,
-            };
-
-            // Now we can intern the variant layouts and store them in the enum layout.
-            best_layout.layout.variants = match best_layout.layout.variants {
-                Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
-                    tag,
-                    tag_encoding,
-                    tag_field,
-                    variants: best_layout
-                        .variants
-                        .into_iter()
-                        .map(|layout| tcx.intern_layout(layout))
-                        .collect(),
-                },
-                _ => bug!(),
-            };
-
-            tcx.intern_layout(best_layout.layout)
+                    },
+                )
+                .ok_or(LayoutError::SizeOverflow(ty))?,
+            )
         }
 
         // Types with no meaningful known layout.
@@ -1488,8 +597,8 @@ fn generator_layout<'tcx>(
     let subst_field = |ty: Ty<'tcx>| EarlyBinder(ty).subst(tcx, substs);
 
     let Some(info) = tcx.generator_layout(def_id) else {
-            return Err(LayoutError::Unknown(ty));
-        };
+        return Err(LayoutError::Unknown(ty));
+    };
     let (ineligible_locals, assignments) = generator_saved_local_eligibility(&info);
 
     // Build a prefix layout, including "promoting" all ineligible
@@ -1592,8 +701,8 @@ fn generator_layout<'tcx>(
             variant.variants = Variants::Single { index };
 
             let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else {
-                    bug!();
-                };
+                bug!();
+            };
 
             // Now, stitch the promoted and variant-only fields back together in
             // the order they are mentioned by our GeneratorLayout.
@@ -1640,13 +749,13 @@ fn generator_layout<'tcx>(
 
             size = size.max(variant.size);
             align = align.max(variant.align);
-            Ok(tcx.intern_layout(variant))
+            Ok(variant)
         })
         .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
 
     size = size.align_to(align.abi);
 
-    let abi = if prefix.abi.is_uninhabited() || variants.iter().all(|v| v.abi().is_uninhabited()) {
+    let abi = if prefix.abi.is_uninhabited() || variants.iter().all(|v| v.abi.is_uninhabited()) {
         Abi::Uninhabited
     } else {
         Abi::Aggregate { sized: true }
diff --git a/compiler/rustc_ty_utils/src/layout_sanity_check.rs b/compiler/rustc_ty_utils/src/layout_sanity_check.rs
index 100926ad4..a5311dbd1 100644
--- a/compiler/rustc_ty_utils/src/layout_sanity_check.rs
+++ b/compiler/rustc_ty_utils/src/layout_sanity_check.rs
@@ -12,7 +12,7 @@ pub(super) fn sanity_check_layout<'tcx>(
     layout: &TyAndLayout<'tcx>,
 ) {
     // Type-level uninhabitedness should always imply ABI uninhabitedness.
-    if cx.tcx.conservative_is_privately_uninhabited(cx.param_env.and(layout.ty)) {
+    if layout.ty.is_privately_uninhabited(cx.tcx, cx.param_env) {
         assert!(layout.abi.is_uninhabited());
     }
 
@@ -20,283 +20,293 @@ pub(super) fn sanity_check_layout<'tcx>(
         bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
     }
 
-    if cfg!(debug_assertions) {
-        /// Yields non-ZST fields of the type
-        fn non_zst_fields<'tcx, 'a>(
-            cx: &'a LayoutCx<'tcx, TyCtxt<'tcx>>,
-            layout: &'a TyAndLayout<'tcx>,
-        ) -> impl Iterator<Item = (Size, TyAndLayout<'tcx>)> + 'a {
-            (0..layout.layout.fields().count()).filter_map(|i| {
-                let field = layout.field(cx, i);
-                // Also checking `align == 1` here leads to test failures in
-                // `layout/zero-sized-array-union.rs`, where a type has a zero-size field with
-                // alignment 4 that still gets ignored during layout computation (which is okay
-                // since other fields already force alignment 4).
-                let zst = field.is_zst();
-                (!zst).then(|| (layout.fields.offset(i), field))
-            })
-        }
+    if !cfg!(debug_assertions) {
+        // Stop here, the rest is kind of expensive.
+        return;
+    }
 
-        fn skip_newtypes<'tcx>(
-            cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
-            layout: &TyAndLayout<'tcx>,
-        ) -> TyAndLayout<'tcx> {
-            if matches!(layout.layout.variants(), Variants::Multiple { .. }) {
-                // Definitely not a newtype of anything.
-                return *layout;
-            }
-            let mut fields = non_zst_fields(cx, layout);
-            let Some(first) = fields.next() else {
-                // No fields here, so this could be a primitive or enum -- either way it's not a newtype around a thing
-                return *layout
-            };
-            if fields.next().is_none() {
-                let (offset, first) = first;
-                if offset == Size::ZERO && first.layout.size() == layout.size {
-                    // This is a newtype, so keep recursing.
-                    // FIXME(RalfJung): I don't think it would be correct to do any checks for
-                    // alignment here, so we don't. Is that correct?
-                    return skip_newtypes(cx, &first);
-                }
+    /// Yields non-ZST fields of the type
+    fn non_zst_fields<'tcx, 'a>(
+        cx: &'a LayoutCx<'tcx, TyCtxt<'tcx>>,
+        layout: &'a TyAndLayout<'tcx>,
+    ) -> impl Iterator<Item = (Size, TyAndLayout<'tcx>)> + 'a {
+        (0..layout.layout.fields().count()).filter_map(|i| {
+            let field = layout.field(cx, i);
+            // Also checking `align == 1` here leads to test failures in
+            // `layout/zero-sized-array-union.rs`, where a type has a zero-size field with
+            // alignment 4 that still gets ignored during layout computation (which is okay
+            // since other fields already force alignment 4).
+            let zst = field.is_zst();
+            (!zst).then(|| (layout.fields.offset(i), field))
+        })
+    }
+
+    fn skip_newtypes<'tcx>(
+        cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+        layout: &TyAndLayout<'tcx>,
+    ) -> TyAndLayout<'tcx> {
+        if matches!(layout.layout.variants(), Variants::Multiple { .. }) {
+            // Definitely not a newtype of anything.
+            return *layout;
+        }
+        let mut fields = non_zst_fields(cx, layout);
+        let Some(first) = fields.next() else {
+            // No fields here, so this could be a primitive or enum -- either way it's not a newtype around a thing
+            return *layout
+        };
+        if fields.next().is_none() {
+            let (offset, first) = first;
+            if offset == Size::ZERO && first.layout.size() == layout.size {
+                // This is a newtype, so keep recursing.
+                // FIXME(RalfJung): I don't think it would be correct to do any checks for
+                // alignment here, so we don't. Is that correct?
+                return skip_newtypes(cx, &first);
             }
-            // No more newtypes here.
-            *layout
         }
+        // No more newtypes here.
+        *layout
+    }
 
-        fn check_layout_abi<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>) {
-            match layout.layout.abi() {
-                Abi::Scalar(scalar) => {
-                    // No padding in scalars.
-                    let size = scalar.size(cx);
-                    let align = scalar.align(cx).abi;
-                    assert_eq!(
-                        layout.layout.size(),
-                        size,
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                    assert_eq!(
-                        layout.layout.align().abi,
-                        align,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    // Check that this matches the underlying field.
-                    let inner = skip_newtypes(cx, layout);
-                    assert!(
-                        matches!(inner.layout.abi(), Abi::Scalar(_)),
-                        "`Scalar` type {} is newtype around non-`Scalar` type {}",
-                        layout.ty,
-                        inner.ty
-                    );
-                    match inner.layout.fields() {
-                        FieldsShape::Primitive => {
-                            // Fine.
-                        }
-                        FieldsShape::Union(..) => {
-                            // FIXME: I guess we could also check something here? Like, look at all fields?
-                            return;
-                        }
-                        FieldsShape::Arbitrary { .. } => {
-                            // Should be an enum, the only field is the discriminant.
-                            assert!(
-                                inner.ty.is_enum(),
-                                "`Scalar` layout for non-primitive non-enum type {}",
-                                inner.ty
-                            );
-                            assert_eq!(
-                                inner.layout.fields().count(),
-                                1,
-                                "`Scalar` layout for multiple-field type in {inner:#?}",
-                            );
-                            let offset = inner.layout.fields().offset(0);
-                            let field = inner.field(cx, 0);
-                            // The field should be at the right offset, and match the `scalar` layout.
-                            assert_eq!(
-                                offset,
-                                Size::ZERO,
-                                "`Scalar` field at non-0 offset in {inner:#?}",
-                            );
-                            assert_eq!(
-                                field.size, size,
-                                "`Scalar` field with bad size in {inner:#?}",
-                            );
-                            assert_eq!(
-                                field.align.abi, align,
-                                "`Scalar` field with bad align in {inner:#?}",
-                            );
-                            assert!(
-                                matches!(field.abi, Abi::Scalar(_)),
-                                "`Scalar` field with bad ABI in {inner:#?}",
-                            );
-                        }
-                        _ => {
-                            panic!("`Scalar` layout for non-primitive non-enum type {}", inner.ty);
-                        }
+    fn check_layout_abi<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>) {
+        match layout.layout.abi() {
+            Abi::Scalar(scalar) => {
+                // No padding in scalars.
+                let size = scalar.size(cx);
+                let align = scalar.align(cx).abi;
+                assert_eq!(
+                    layout.layout.size(),
+                    size,
+                    "size mismatch between ABI and layout in {layout:#?}"
+                );
+                assert_eq!(
+                    layout.layout.align().abi,
+                    align,
+                    "alignment mismatch between ABI and layout in {layout:#?}"
+                );
+                // Check that this matches the underlying field.
+                let inner = skip_newtypes(cx, layout);
+                assert!(
+                    matches!(inner.layout.abi(), Abi::Scalar(_)),
+                    "`Scalar` type {} is newtype around non-`Scalar` type {}",
+                    layout.ty,
+                    inner.ty
+                );
+                match inner.layout.fields() {
+                    FieldsShape::Primitive => {
+                        // Fine.
                     }
-                }
-                Abi::ScalarPair(scalar1, scalar2) => {
-                    // Sanity-check scalar pairs. These are a bit more flexible and support
-                    // padding, but we can at least ensure both fields actually fit into the layout
-                    // and the alignment requirement has not been weakened.
-                    let size1 = scalar1.size(cx);
-                    let align1 = scalar1.align(cx).abi;
-                    let size2 = scalar2.size(cx);
-                    let align2 = scalar2.align(cx).abi;
-                    assert!(
-                        layout.layout.align().abi >= cmp::max(align1, align2),
-                        "alignment mismatch between ABI and layout in {layout:#?}",
-                    );
-                    let field2_offset = size1.align_to(align2);
-                    assert!(
-                        layout.layout.size() >= field2_offset + size2,
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                    // Check that the underlying pair of fields matches.
-                    let inner = skip_newtypes(cx, layout);
-                    assert!(
-                        matches!(inner.layout.abi(), Abi::ScalarPair(..)),
-                        "`ScalarPair` type {} is newtype around non-`ScalarPair` type {}",
-                        layout.ty,
-                        inner.ty
-                    );
-                    if matches!(inner.layout.variants(), Variants::Multiple { .. }) {
-                        // FIXME: ScalarPair for enums is enormously complicated and it is very hard
-                        // to check anything about them.
+                    FieldsShape::Union(..) => {
+                        // FIXME: I guess we could also check something here? Like, look at all fields?
                         return;
                     }
-                    match inner.layout.fields() {
-                        FieldsShape::Arbitrary { .. } => {
-                            // Checked below.
-                        }
-                        FieldsShape::Union(..) => {
-                            // FIXME: I guess we could also check something here? Like, look at all fields?
-                            return;
-                        }
-                        _ => {
-                            panic!("`ScalarPair` layout with unexpected field shape in {inner:#?}");
-                        }
+                    FieldsShape::Arbitrary { .. } => {
+                        // Should be an enum, the only field is the discriminant.
+                        assert!(
+                            inner.ty.is_enum(),
+                            "`Scalar` layout for non-primitive non-enum type {}",
+                            inner.ty
+                        );
+                        assert_eq!(
+                            inner.layout.fields().count(),
+                            1,
+                            "`Scalar` layout for multiple-field type in {inner:#?}",
+                        );
+                        let offset = inner.layout.fields().offset(0);
+                        let field = inner.field(cx, 0);
+                        // The field should be at the right offset, and match the `scalar` layout.
+                        assert_eq!(
+                            offset,
+                            Size::ZERO,
+                            "`Scalar` field at non-0 offset in {inner:#?}",
+                        );
+                        assert_eq!(field.size, size, "`Scalar` field with bad size in {inner:#?}",);
+                        assert_eq!(
+                            field.align.abi, align,
+                            "`Scalar` field with bad align in {inner:#?}",
+                        );
+                        assert!(
+                            matches!(field.abi, Abi::Scalar(_)),
+                            "`Scalar` field with bad ABI in {inner:#?}",
+                        );
+                    }
+                    _ => {
+                        panic!("`Scalar` layout for non-primitive non-enum type {}", inner.ty);
                     }
-                    let mut fields = non_zst_fields(cx, &inner);
-                    let (offset1, field1) = fields.next().unwrap_or_else(|| {
-                        panic!("`ScalarPair` layout for type with not even one non-ZST field: {inner:#?}")
-                    });
-                    let (offset2, field2) = fields.next().unwrap_or_else(|| {
-                        panic!("`ScalarPair` layout for type with less than two non-ZST fields: {inner:#?}")
-                    });
-                    assert!(
-                        fields.next().is_none(),
-                        "`ScalarPair` layout for type with at least three non-ZST fields: {inner:#?}"
-                    );
-                    // The fields might be in opposite order.
-                    let (offset1, field1, offset2, field2) = if offset1 <= offset2 {
-                        (offset1, field1, offset2, field2)
-                    } else {
-                        (offset2, field2, offset1, field1)
-                    };
-                    // The fields should be at the right offset, and match the `scalar` layout.
-                    assert_eq!(
-                        offset1,
-                        Size::ZERO,
-                        "`ScalarPair` first field at non-0 offset in {inner:#?}",
-                    );
-                    assert_eq!(
-                        field1.size, size1,
-                        "`ScalarPair` first field with bad size in {inner:#?}",
-                    );
-                    assert_eq!(
-                        field1.align.abi, align1,
-                        "`ScalarPair` first field with bad align in {inner:#?}",
-                    );
-                    assert!(
-                        matches!(field1.abi, Abi::Scalar(_)),
-                        "`ScalarPair` first field with bad ABI in {inner:#?}",
-                    );
-                    assert_eq!(
-                        offset2, field2_offset,
-                        "`ScalarPair` second field at bad offset in {inner:#?}",
-                    );
-                    assert_eq!(
-                        field2.size, size2,
-                        "`ScalarPair` second field with bad size in {inner:#?}",
-                    );
-                    assert_eq!(
-                        field2.align.abi, align2,
-                        "`ScalarPair` second field with bad align in {inner:#?}",
-                    );
-                    assert!(
-                        matches!(field2.abi, Abi::Scalar(_)),
-                        "`ScalarPair` second field with bad ABI in {inner:#?}",
-                    );
                 }
-                Abi::Vector { count, element } => {
-                    // No padding in vectors. Alignment can be strengthened, though.
-                    assert!(
-                        layout.layout.align().abi >= element.align(cx).abi,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    let size = element.size(cx) * count;
-                    assert_eq!(
-                        layout.layout.size(),
-                        size.align_to(cx.data_layout().vector_align(size).abi),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
+            }
+            Abi::ScalarPair(scalar1, scalar2) => {
+                // Sanity-check scalar pairs. Computing the expected size and alignment is a bit of work.
+                let size1 = scalar1.size(cx);
+                let align1 = scalar1.align(cx).abi;
+                let size2 = scalar2.size(cx);
+                let align2 = scalar2.align(cx).abi;
+                let align = cmp::max(align1, align2);
+                let field2_offset = size1.align_to(align2);
+                let size = (field2_offset + size2).align_to(align);
+                assert_eq!(
+                    layout.layout.size(),
+                    size,
+                    "size mismatch between ABI and layout in {layout:#?}"
+                );
+                assert_eq!(
+                    layout.layout.align().abi,
+                    align,
+                    "alignment mismatch between ABI and layout in {layout:#?}",
+                );
+                // Check that the underlying pair of fields matches.
+                let inner = skip_newtypes(cx, layout);
+                assert!(
+                    matches!(inner.layout.abi(), Abi::ScalarPair(..)),
+                    "`ScalarPair` type {} is newtype around non-`ScalarPair` type {}",
+                    layout.ty,
+                    inner.ty
+                );
+                if matches!(inner.layout.variants(), Variants::Multiple { .. }) {
+                    // FIXME: ScalarPair for enums is enormously complicated and it is very hard
+                    // to check anything about them.
+                    return;
+                }
+                match inner.layout.fields() {
+                    FieldsShape::Arbitrary { .. } => {
+                        // Checked below.
+                    }
+                    FieldsShape::Union(..) => {
+                        // FIXME: I guess we could also check something here? Like, look at all fields?
+                        return;
+                    }
+                    _ => {
+                        panic!("`ScalarPair` layout with unexpected field shape in {inner:#?}");
+                    }
                 }
-                Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
+                let mut fields = non_zst_fields(cx, &inner);
+                let (offset1, field1) = fields.next().unwrap_or_else(|| {
+                    panic!(
+                        "`ScalarPair` layout for type with not even one non-ZST field: {inner:#?}"
+                    )
+                });
+                let (offset2, field2) = fields.next().unwrap_or_else(|| {
+                    panic!(
+                        "`ScalarPair` layout for type with less than two non-ZST fields: {inner:#?}"
+                    )
+                });
+                assert!(
+                    fields.next().is_none(),
+                    "`ScalarPair` layout for type with at least three non-ZST fields: {inner:#?}"
+                );
+                // The fields might be in opposite order.
+                let (offset1, field1, offset2, field2) = if offset1 <= offset2 {
+                    (offset1, field1, offset2, field2)
+                } else {
+                    (offset2, field2, offset1, field1)
+                };
+                // The fields should be at the right offset, and match the `scalar` layout.
+                assert_eq!(
+                    offset1,
+                    Size::ZERO,
+                    "`ScalarPair` first field at non-0 offset in {inner:#?}",
+                );
+                assert_eq!(
+                    field1.size, size1,
+                    "`ScalarPair` first field with bad size in {inner:#?}",
+                );
+                assert_eq!(
+                    field1.align.abi, align1,
+                    "`ScalarPair` first field with bad align in {inner:#?}",
+                );
+                assert!(
+                    matches!(field1.abi, Abi::Scalar(_)),
+                    "`ScalarPair` first field with bad ABI in {inner:#?}",
+                );
+                assert_eq!(
+                    offset2, field2_offset,
+                    "`ScalarPair` second field at bad offset in {inner:#?}",
+                );
+                assert_eq!(
+                    field2.size, size2,
+                    "`ScalarPair` second field with bad size in {inner:#?}",
+                );
+                assert_eq!(
+                    field2.align.abi, align2,
+                    "`ScalarPair` second field with bad align in {inner:#?}",
+                );
+                assert!(
+                    matches!(field2.abi, Abi::Scalar(_)),
+                    "`ScalarPair` second field with bad ABI in {inner:#?}",
+                );
             }
+            Abi::Vector { count, element } => {
+                // No padding in vectors, except possibly for trailing padding to make the size a multiple of align.
+                let size = element.size(cx) * count;
+                let align = cx.data_layout().vector_align(size).abi;
+                let size = size.align_to(align); // needed e.g. for vectors of size 3
+                assert!(align >= element.align(cx).abi); // just sanity-checking `vector_align`.
+                assert_eq!(
+                    layout.layout.size(),
+                    size,
+                    "size mismatch between ABI and layout in {layout:#?}"
+                );
+                assert_eq!(
+                    layout.layout.align().abi,
+                    align,
+                    "alignment mismatch between ABI and layout in {layout:#?}"
+                );
+                // FIXME: Do some kind of check of the inner type, like for Scalar and ScalarPair.
+            }
+            Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
         }
+    }
 
-        check_layout_abi(cx, layout);
+    check_layout_abi(cx, layout);
 
-        if let Variants::Multiple { variants, .. } = &layout.variants {
-            for variant in variants.iter() {
-                // No nested "multiple".
-                assert!(matches!(variant.variants(), Variants::Single { .. }));
-                // Variants should have the same or a smaller size as the full thing,
-                // and same for alignment.
-                if variant.size() > layout.size {
-                    bug!(
-                        "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
-                        layout.size.bytes(),
-                        variant.size().bytes(),
-                    )
-                }
-                if variant.align().abi > layout.align.abi {
-                    bug!(
-                        "Type with alignment {} bytes has variant with alignment {} bytes: {layout:#?}",
-                        layout.align.abi.bytes(),
-                        variant.align().abi.bytes(),
-                    )
-                }
-                // Skip empty variants.
-                if variant.size() == Size::ZERO
-                    || variant.fields().count() == 0
-                    || variant.abi().is_uninhabited()
-                {
-                    // These are never actually accessed anyway, so we can skip the coherence check
-                    // for them. They also fail that check, since they have
-                    // `Aggregate`/`Uninhbaited` ABI even when the main type is
-                    // `Scalar`/`ScalarPair`. (Note that sometimes, variants with fields have size
-                    // 0, and sometimes, variants without fields have non-0 size.)
-                    continue;
-                }
-                // The top-level ABI and the ABI of the variants should be coherent.
-                let scalar_coherent = |s1: Scalar, s2: Scalar| {
-                    s1.size(cx) == s2.size(cx) && s1.align(cx) == s2.align(cx)
-                };
-                let abi_coherent = match (layout.abi, variant.abi()) {
-                    (Abi::Scalar(s1), Abi::Scalar(s2)) => scalar_coherent(s1, s2),
-                    (Abi::ScalarPair(a1, b1), Abi::ScalarPair(a2, b2)) => {
-                        scalar_coherent(a1, a2) && scalar_coherent(b1, b2)
-                    }
-                    (Abi::Uninhabited, _) => true,
-                    (Abi::Aggregate { .. }, _) => true,
-                    _ => false,
-                };
-                if !abi_coherent {
-                    bug!(
-                        "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
-                        variant
-                    );
+    if let Variants::Multiple { variants, .. } = &layout.variants {
+        for variant in variants.iter() {
+            // No nested "multiple".
+            assert!(matches!(variant.variants, Variants::Single { .. }));
+            // Variants should have the same or a smaller size as the full thing,
+            // and same for alignment.
+            if variant.size > layout.size {
+                bug!(
+                    "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
+                    layout.size.bytes(),
+                    variant.size.bytes(),
+                )
+            }
+            if variant.align.abi > layout.align.abi {
+                bug!(
+                    "Type with alignment {} bytes has variant with alignment {} bytes: {layout:#?}",
+                    layout.align.abi.bytes(),
+                    variant.align.abi.bytes(),
+                )
+            }
+            // Skip empty variants.
+            if variant.size == Size::ZERO
+                || variant.fields.count() == 0
+                || variant.abi.is_uninhabited()
+            {
+                // These are never actually accessed anyway, so we can skip the coherence check
+                // for them. They also fail that check, since they have
+                // `Aggregate`/`Uninhbaited` ABI even when the main type is
+                // `Scalar`/`ScalarPair`. (Note that sometimes, variants with fields have size
+                // 0, and sometimes, variants without fields have non-0 size.)
+                continue;
+            }
+            // The top-level ABI and the ABI of the variants should be coherent.
+            let scalar_coherent =
+                |s1: Scalar, s2: Scalar| s1.size(cx) == s2.size(cx) && s1.align(cx) == s2.align(cx);
+            let abi_coherent = match (layout.abi, variant.abi) {
+                (Abi::Scalar(s1), Abi::Scalar(s2)) => scalar_coherent(s1, s2),
+                (Abi::ScalarPair(a1, b1), Abi::ScalarPair(a2, b2)) => {
+                    scalar_coherent(a1, a2) && scalar_coherent(b1, b2)
                 }
+                (Abi::Uninhabited, _) => true,
+                (Abi::Aggregate { .. }, _) => true,
+                _ => false,
+            };
+            if !abi_coherent {
+                bug!(
+                    "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
+                    variant
+                );
             }
         }
     }
diff --git a/compiler/rustc_ty_utils/src/lib.rs b/compiler/rustc_ty_utils/src/lib.rs
index cce5a79dd..7ad5cbc01 100644
--- a/compiler/rustc_ty_utils/src/lib.rs
+++ b/compiler/rustc_ty_utils/src/lib.rs
@@ -29,6 +29,7 @@ mod layout;
 mod layout_sanity_check;
 mod needs_drop;
 pub mod representability;
+mod structural_match;
 mod ty;
 
 pub fn provide(providers: &mut Providers) {
@@ -42,4 +43,5 @@ pub fn provide(providers: &mut Providers) {
     representability::provide(providers);
     ty::provide(providers);
     instance::provide(providers);
+    structural_match::provide(providers);
 }
diff --git a/compiler/rustc_ty_utils/src/structural_match.rs b/compiler/rustc_ty_utils/src/structural_match.rs
new file mode 100644
index 000000000..a55bb7e7e
--- /dev/null
+++ b/compiler/rustc_ty_utils/src/structural_match.rs
@@ -0,0 +1,44 @@
+use rustc_hir::lang_items::LangItem;
+use rustc_middle::ty::query::Providers;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+
+use rustc_infer::infer::TyCtxtInferExt;
+use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt};
+
+/// This method returns true if and only if `adt_ty` itself has been marked as
+/// eligible for structural-match: namely, if it implements both
+/// `StructuralPartialEq` and `StructuralEq` (which are respectively injected by
+/// `#[derive(PartialEq)]` and `#[derive(Eq)]`).
+///
+/// Note that this does *not* recursively check if the substructure of `adt_ty`
+/// implements the traits.
+fn has_structural_eq_impls<'tcx>(tcx: TyCtxt<'tcx>, adt_ty: Ty<'tcx>) -> bool {
+    let ref infcx = tcx.infer_ctxt().build();
+    let cause = ObligationCause::dummy();
+
+    let ocx = ObligationCtxt::new(infcx);
+    // require `#[derive(PartialEq)]`
+    let structural_peq_def_id =
+        infcx.tcx.require_lang_item(LangItem::StructuralPeq, Some(cause.span));
+    ocx.register_bound(cause.clone(), ty::ParamEnv::empty(), adt_ty, structural_peq_def_id);
+    // for now, require `#[derive(Eq)]`. (Doing so is a hack to work around
+    // the type `for<'a> fn(&'a ())` failing to implement `Eq` itself.)
+    let structural_teq_def_id =
+        infcx.tcx.require_lang_item(LangItem::StructuralTeq, Some(cause.span));
+    ocx.register_bound(cause, ty::ParamEnv::empty(), adt_ty, structural_teq_def_id);
+
+    // We deliberately skip *reporting* fulfillment errors (via
+    // `report_fulfillment_errors`), for two reasons:
+    //
+    // 1. The error messages would mention `std::marker::StructuralPartialEq`
+    //    (a trait which is solely meant as an implementation detail
+    //    for now), and
+    //
+    // 2. We are sometimes doing future-incompatibility lints for
+    //    now, so we do not want unconditional errors here.
+    ocx.select_all_or_error().is_empty()
+}
+
+pub fn provide(providers: &mut Providers) {
+    providers.has_structural_eq_impls = has_structural_eq_impls;
+}
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index 3eebb4ace..5fc9bcac1 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -49,12 +49,9 @@ fn sized_constraint_for_ty<'tcx>(
             // it on the impl.
 
             let Some(sized_trait) = tcx.lang_items().sized_trait() else { return vec![ty] };
-            let sized_predicate = ty::Binder::dummy(ty::TraitRef {
-                def_id: sized_trait,
-                substs: tcx.mk_substs_trait(ty, &[]),
-            })
-            .without_const()
-            .to_predicate(tcx);
+            let sized_predicate = ty::Binder::dummy(tcx.mk_trait_ref(sized_trait, [ty]))
+                .without_const()
+                .to_predicate(tcx);
             let predicates = tcx.predicates_of(adtdef.did()).predicates;
             if predicates.iter().any(|(p, _)| *p == sized_predicate) { vec![] } else { vec![ty] }
         }
@@ -108,12 +105,7 @@ fn adt_sized_constraint(tcx: TyCtxt<'_>, def_id: DefId) -> &[Ty<'_>] {
 
 /// See `ParamEnv` struct definition for details.
 fn param_env(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
-    // The param_env of an impl Trait type is its defining function's param_env
-    if let Some(parent) = ty::is_impl_trait_defn(tcx, def_id) {
-        return param_env(tcx, parent.to_def_id());
-    }
     // Compute the bounds on Self and the type parameters.
-
     let ty::InstantiatedPredicates { mut predicates, .. } =
         tcx.predicates_of(def_id).instantiate_identity(tcx);
 
@@ -413,63 +405,7 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Ty<'_>> {
 /// Check if a function is async.
 fn asyncness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::IsAsync {
     let node = tcx.hir().get_by_def_id(def_id.expect_local());
-    if let Some(fn_kind) = node.fn_kind() { fn_kind.asyncness() } else { hir::IsAsync::NotAsync }
-}
-
-/// Don't call this directly: use ``tcx.conservative_is_privately_uninhabited`` instead.
-pub fn conservative_is_privately_uninhabited_raw<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    param_env_and: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
-) -> bool {
-    let (param_env, ty) = param_env_and.into_parts();
-    match ty.kind() {
-        ty::Never => {
-            debug!("ty::Never =>");
-            true
-        }
-        ty::Adt(def, _) if def.is_union() => {
-            debug!("ty::Adt(def, _) if def.is_union() =>");
-            // For now, `union`s are never considered uninhabited.
-            false
-        }
-        ty::Adt(def, substs) => {
-            debug!("ty::Adt(def, _) if def.is_not_union() =>");
-            // Any ADT is uninhabited if either:
-            // (a) It has no variants (i.e. an empty `enum`);
-            // (b) Each of its variants (a single one in the case of a `struct`) has at least
-            //     one uninhabited field.
-            def.variants().iter().all(|var| {
-                var.fields.iter().any(|field| {
-                    let ty = tcx.bound_type_of(field.did).subst(tcx, substs);
-                    tcx.conservative_is_privately_uninhabited(param_env.and(ty))
-                })
-            })
-        }
-        ty::Tuple(fields) => {
-            debug!("ty::Tuple(..) =>");
-            fields.iter().any(|ty| tcx.conservative_is_privately_uninhabited(param_env.and(ty)))
-        }
-        ty::Array(ty, len) => {
-            debug!("ty::Array(ty, len) =>");
-            match len.try_eval_usize(tcx, param_env) {
-                Some(0) | None => false,
-                // If the array is definitely non-empty, it's uninhabited if
-                // the type of its elements is uninhabited.
-                Some(1..) => tcx.conservative_is_privately_uninhabited(param_env.and(*ty)),
-            }
-        }
-        ty::Ref(..) => {
-            debug!("ty::Ref(..) =>");
-            // References to uninitialised memory is valid for any type, including
-            // uninhabited types, in unsafe code, so we treat all references as
-            // inhabited.
-            false
-        }
-        _ => {
-            debug!("_ =>");
-            false
-        }
-    }
+    node.fn_sig().map_or(hir::IsAsync::NotAsync, |sig| sig.header.asyncness)
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {
@@ -481,7 +417,6 @@ pub fn provide(providers: &mut ty::query::Providers) {
         instance_def_size_estimate,
         issue33140_self_ty,
         impl_defaultness,
-        conservative_is_privately_uninhabited: conservative_is_privately_uninhabited_raw,
         ..*providers
     };
 }