Merging upstream version 1.75.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 782 insertions, 354 deletions

diff --git a/compiler/rustc_smir/Cargo.toml b/compiler/rustc_smir/Cargo.toml
index 4c29f7437..836ea046f 100644
--- a/compiler/rustc_smir/Cargo.toml
+++ b/compiler/rustc_smir/Cargo.toml
@@ -4,14 +4,13 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
+rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
-rustc_driver = { path = "../rustc_driver" }
-rustc_interface = { path = "../rustc_interface" }
-rustc_session = {path = "../rustc_session" }
-tracing = "0.1"
+scoped-tls = "1.0"
 stable_mir = {path = "../stable_mir" }
-
-[features]
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_smir/src/lib.rs b/compiler/rustc_smir/src/lib.rs
index b6c36678d..dcf6b9040 100644
--- a/compiler/rustc_smir/src/lib.rs
+++ b/compiler/rustc_smir/src/lib.rs
@@ -10,10 +10,10 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(allow(unused_variables), deny(warnings)))
 )]
-#![feature(rustc_private)]
-#![feature(ptr_metadata)]
-#![feature(type_alias_impl_trait)] // Used to define opaque types.
-#![feature(intra_doc_pointers)]
+#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![allow(rustc::usage_of_ty_tykind)]
 
 pub mod rustc_internal;
 
diff --git a/compiler/rustc_smir/src/rustc_internal/internal.rs b/compiler/rustc_smir/src/rustc_internal/internal.rs
new file mode 100644
index 000000000..7cfdbbbf7
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_internal/internal.rs
@@ -0,0 +1,67 @@
+//! Module containing the translation from stable mir constructs to the rustc counterpart.
+//!
+//! This module will only include a few constructs to allow users to invoke internal rustc APIs
+//! due to incomplete stable coverage.
+
+// Prefer importing stable_mir over internal rustc constructs to make this file more readable.
+use crate::rustc_smir::Tables;
+use rustc_middle::ty::{self as rustc_ty, Ty as InternalTy};
+use stable_mir::ty::{Const, GenericArgKind, GenericArgs, Region, Ty};
+use stable_mir::DefId;
+
+use super::RustcInternal;
+
+impl<'tcx> RustcInternal<'tcx> for DefId {
+    type T = rustc_span::def_id::DefId;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.def_ids[*self]
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for GenericArgs {
+    type T = rustc_ty::GenericArgsRef<'tcx>;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.tcx.mk_args_from_iter(self.0.iter().map(|arg| arg.internal(tables)))
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for GenericArgKind {
+    type T = rustc_ty::GenericArg<'tcx>;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            GenericArgKind::Lifetime(reg) => reg.internal(tables).into(),
+            GenericArgKind::Type(ty) => ty.internal(tables).into(),
+            GenericArgKind::Const(cnst) => ty_const(cnst, tables).into(),
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Region {
+    type T = rustc_ty::Region<'tcx>;
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        todo!()
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Ty {
+    type T = InternalTy<'tcx>;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.types[*self]
+    }
+}
+
+fn ty_const<'tcx>(constant: &Const, tables: &mut Tables<'tcx>) -> rustc_ty::Const<'tcx> {
+    match constant.internal(tables) {
+        rustc_middle::mir::Const::Ty(c) => c,
+        cnst => {
+            panic!("Trying to convert constant `{constant:?}` to type constant, but found {cnst:?}")
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Const {
+    type T = rustc_middle::mir::Const<'tcx>;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.constants[self.id]
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs
index 1a9dea99f..4d2a51822 100644
--- a/compiler/rustc_smir/src/rustc_internal/mod.rs
+++ b/compiler/rustc_smir/src/rustc_internal/mod.rs
@@ -3,24 +3,38 @@
 //! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs
 //! until stable MIR is complete.
 
-use std::ops::{ControlFlow, Index};
-
-use crate::rustc_internal;
-use crate::rustc_smir::Tables;
-use rustc_driver::{Callbacks, Compilation, RunCompiler};
-use rustc_interface::{interface, Queries};
+use crate::rustc_smir::{Stable, Tables, TablesWrapper};
+use rustc_data_structures::fx;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_middle::mir::interpret::AllocId;
+use rustc_middle::ty;
 use rustc_middle::ty::TyCtxt;
-pub use rustc_span::def_id::{CrateNum, DefId};
+use rustc_span::def_id::{CrateNum, DefId};
 use rustc_span::Span;
-use stable_mir::CompilerError;
+use scoped_tls::scoped_thread_local;
+use stable_mir::ty::IndexedVal;
+use std::cell::Cell;
+use std::cell::RefCell;
+use std::fmt::Debug;
+use std::hash::Hash;
+use std::ops::Index;
+
+mod internal;
+
+pub fn stable<'tcx, S: Stable<'tcx>>(item: &S) -> S::T {
+    with_tables(|tables| item.stable(tables))
+}
+
+pub fn internal<'tcx, S: RustcInternal<'tcx>>(item: &S) -> S::T {
+    with_tables(|tables| item.internal(tables))
+}
 
 impl<'tcx> Index<stable_mir::DefId> for Tables<'tcx> {
     type Output = DefId;
 
     #[inline(always)]
     fn index(&self, index: stable_mir::DefId) -> &Self::Output {
-        &self.def_ids[index.0]
+        &self.def_ids[index]
     }
 }
 
@@ -29,7 +43,7 @@ impl<'tcx> Index<stable_mir::ty::Span> for Tables<'tcx> {
 
     #[inline(always)]
     fn index(&self, index: stable_mir::ty::Span) -> &Self::Output {
-        &self.spans[index.0]
+        &self.spans[index]
     }
 }
 
@@ -54,8 +68,8 @@ impl<'tcx> Tables<'tcx> {
         stable_mir::ty::ClosureDef(self.create_def_id(did))
     }
 
-    pub fn generator_def(&mut self, did: DefId) -> stable_mir::ty::GeneratorDef {
-        stable_mir::ty::GeneratorDef(self.create_def_id(did))
+    pub fn coroutine_def(&mut self, did: DefId) -> stable_mir::ty::CoroutineDef {
+        stable_mir::ty::CoroutineDef(self.create_def_id(did))
     }
 
     pub fn alias_def(&mut self, did: DefId) -> stable_mir::ty::AliasDef {
@@ -94,37 +108,27 @@ impl<'tcx> Tables<'tcx> {
         stable_mir::ty::Prov(self.create_alloc_id(aid))
     }
 
-    fn create_def_id(&mut self, did: DefId) -> stable_mir::DefId {
-        // FIXME: this becomes inefficient when we have too many ids
-        for (i, &d) in self.def_ids.iter().enumerate() {
-            if d == did {
-                return stable_mir::DefId(i);
-            }
-        }
-        let id = self.def_ids.len();
-        self.def_ids.push(did);
-        stable_mir::DefId(id)
+    pub(crate) fn create_def_id(&mut self, did: DefId) -> stable_mir::DefId {
+        self.def_ids.create_or_fetch(did)
     }
 
     fn create_alloc_id(&mut self, aid: AllocId) -> stable_mir::AllocId {
-        // FIXME: this becomes inefficient when we have too many ids
-        if let Some(i) = self.alloc_ids.iter().position(|a| *a == aid) {
-            return stable_mir::AllocId(i);
-        };
-        let id = self.def_ids.len();
-        self.alloc_ids.push(aid);
-        stable_mir::AllocId(id)
+        self.alloc_ids.create_or_fetch(aid)
     }
 
     pub(crate) fn create_span(&mut self, span: Span) -> stable_mir::ty::Span {
-        for (i, &sp) in self.spans.iter().enumerate() {
-            if sp == span {
-                return stable_mir::ty::Span(i);
-            }
-        }
-        let id = self.spans.len();
-        self.spans.push(span);
-        stable_mir::ty::Span(id)
+        self.spans.create_or_fetch(span)
+    }
+
+    pub(crate) fn instance_def(
+        &mut self,
+        instance: ty::Instance<'tcx>,
+    ) -> stable_mir::mir::mono::InstanceDef {
+        self.instances.create_or_fetch(instance)
+    }
+
+    pub(crate) fn static_def(&mut self, did: DefId) -> stable_mir::mir::mono::StaticDef {
+        stable_mir::mir::mono::StaticDef(self.create_def_id(did))
     }
 }
 
@@ -132,68 +136,157 @@ pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
     item.id.into()
 }
 
-pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
-    stable_mir::run(
-        Tables { tcx, def_ids: vec![], alloc_ids: vec![], spans: vec![], types: vec![] },
-        f,
-    );
+// A thread local variable that stores a pointer to the tables mapping between TyCtxt
+// datastructures and stable MIR datastructures
+scoped_thread_local! (static TLV: Cell<*const ()>);
+
+pub(crate) fn init<'tcx>(tables: &TablesWrapper<'tcx>, f: impl FnOnce()) {
+    assert!(!TLV.is_set());
+    let ptr = tables as *const _ as *const ();
+    TLV.set(&Cell::new(ptr), || {
+        f();
+    });
 }
 
-pub struct StableMir<B = (), C = ()>
-where
-    B: Send,
-    C: Send,
-{
-    args: Vec<String>,
-    callback: fn(TyCtxt<'_>) -> ControlFlow<B, C>,
-    result: Option<ControlFlow<B, C>>,
+/// Loads the current context and calls a function with it.
+/// Do not nest these, as that will ICE.
+pub(crate) fn with_tables<'tcx, R>(f: impl FnOnce(&mut Tables<'tcx>) -> R) -> R {
+    assert!(TLV.is_set());
+    TLV.with(|tlv| {
+        let ptr = tlv.get();
+        assert!(!ptr.is_null());
+        let wrapper = ptr as *const TablesWrapper<'tcx>;
+        let mut tables = unsafe { (*wrapper).0.borrow_mut() };
+        f(&mut *tables)
+    })
 }
 
-impl<B, C> StableMir<B, C>
-where
-    B: Send,
-    C: Send,
-{
-    /// Creates a new `StableMir` instance, with given test_function and arguments.
-    pub fn new(args: Vec<String>, callback: fn(TyCtxt<'_>) -> ControlFlow<B, C>) -> Self {
-        StableMir { args, callback, result: None }
-    }
-
-    /// Runs the compiler against given target and tests it with `test_function`
-    pub fn run(&mut self) -> Result<C, CompilerError<B>> {
-        let compiler_result =
-            rustc_driver::catch_fatal_errors(|| RunCompiler::new(&self.args.clone(), self).run());
-        match (compiler_result, self.result.take()) {
-            (Ok(Ok(())), Some(ControlFlow::Continue(value))) => Ok(value),
-            (Ok(Ok(())), Some(ControlFlow::Break(value))) => Err(CompilerError::Interrupted(value)),
-            (Ok(Ok(_)), None) => Err(CompilerError::Skipped),
-            (Ok(Err(_)), _) => Err(CompilerError::CompilationFailed),
-            (Err(_), _) => Err(CompilerError::ICE),
+pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
+    let tables = TablesWrapper(RefCell::new(Tables {
+        tcx,
+        def_ids: IndexMap::default(),
+        alloc_ids: IndexMap::default(),
+        spans: IndexMap::default(),
+        types: IndexMap::default(),
+        instances: IndexMap::default(),
+        constants: IndexMap::default(),
+    }));
+    stable_mir::run(&tables, || init(&tables, f));
+}
+
+#[macro_export]
+macro_rules! run {
+    ($args:expr, $callback:expr) => {
+        run!($args, tcx, $callback)
+    };
+    ($args:expr, $tcx:ident, $callback:expr) => {{
+        use rustc_driver::{Callbacks, Compilation, RunCompiler};
+        use rustc_interface::{interface, Queries};
+        use stable_mir::CompilerError;
+        use std::ops::ControlFlow;
+
+        pub struct StableMir<B = (), C = ()>
+        where
+            B: Send,
+            C: Send,
+        {
+            args: Vec<String>,
+            callback: fn(TyCtxt<'_>) -> ControlFlow<B, C>,
+            result: Option<ControlFlow<B, C>>,
+        }
+
+        impl<B, C> StableMir<B, C>
+        where
+            B: Send,
+            C: Send,
+        {
+            /// Creates a new `StableMir` instance, with given test_function and arguments.
+            pub fn new(args: Vec<String>, callback: fn(TyCtxt<'_>) -> ControlFlow<B, C>) -> Self {
+                StableMir { args, callback, result: None }
+            }
+
+            /// Runs the compiler against given target and tests it with `test_function`
+            pub fn run(&mut self) -> Result<C, CompilerError<B>> {
+                let compiler_result = rustc_driver::catch_fatal_errors(|| {
+                    RunCompiler::new(&self.args.clone(), self).run()
+                });
+                match (compiler_result, self.result.take()) {
+                    (Ok(Ok(())), Some(ControlFlow::Continue(value))) => Ok(value),
+                    (Ok(Ok(())), Some(ControlFlow::Break(value))) => {
+                        Err(CompilerError::Interrupted(value))
+                    }
+                    (Ok(Ok(_)), None) => Err(CompilerError::Skipped),
+                    (Ok(Err(_)), _) => Err(CompilerError::CompilationFailed),
+                    (Err(_), _) => Err(CompilerError::ICE),
+                }
+            }
         }
+
+        impl<B, C> Callbacks for StableMir<B, C>
+        where
+            B: Send,
+            C: Send,
+        {
+            /// Called after analysis. Return value instructs the compiler whether to
+            /// continue the compilation afterwards (defaults to `Compilation::Continue`)
+            fn after_analysis<'tcx>(
+                &mut self,
+                _compiler: &interface::Compiler,
+                queries: &'tcx Queries<'tcx>,
+            ) -> Compilation {
+                queries.global_ctxt().unwrap().enter(|tcx| {
+                    rustc_internal::run(tcx, || {
+                        self.result = Some((self.callback)(tcx));
+                    });
+                    if self.result.as_ref().is_some_and(|val| val.is_continue()) {
+                        Compilation::Continue
+                    } else {
+                        Compilation::Stop
+                    }
+                })
+            }
+        }
+
+        StableMir::new($args, |$tcx| $callback).run()
+    }};
+}
+
+/// Simmilar to rustc's `FxIndexMap`, `IndexMap` with extra
+/// safety features added.
+pub struct IndexMap<K, V> {
+    index_map: fx::FxIndexMap<K, V>,
+}
+
+impl<K, V> Default for IndexMap<K, V> {
+    fn default() -> Self {
+        Self { index_map: FxIndexMap::default() }
+    }
+}
+
+impl<K: PartialEq + Hash + Eq, V: Copy + Debug + PartialEq + IndexedVal> IndexMap<K, V> {
+    pub fn create_or_fetch(&mut self, key: K) -> V {
+        let len = self.index_map.len();
+        let v = self.index_map.entry(key).or_insert(V::to_val(len));
+        *v
     }
 }
 
-impl<B, C> Callbacks for StableMir<B, C>
-where
-    B: Send,
-    C: Send,
+impl<K: PartialEq + Hash + Eq, V: Copy + Debug + PartialEq + IndexedVal> Index<V>
+    for IndexMap<K, V>
 {
-    /// Called after analysis. Return value instructs the compiler whether to
-    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
-    fn after_analysis<'tcx>(
-        &mut self,
-        _compiler: &interface::Compiler,
-        queries: &'tcx Queries<'tcx>,
-    ) -> Compilation {
-        queries.global_ctxt().unwrap().enter(|tcx| {
-            rustc_internal::run(tcx, || {
-                self.result = Some((self.callback)(tcx));
-            });
-            if self.result.as_ref().is_some_and(|val| val.is_continue()) {
-                Compilation::Continue
-            } else {
-                Compilation::Stop
-            }
-        })
+    type Output = K;
+
+    fn index(&self, index: V) -> &Self::Output {
+        let (k, v) = self.index_map.get_index(index.to_index()).unwrap();
+        assert_eq!(*v, index, "Provided value doesn't match with indexed value");
+        k
     }
 }
+
+/// Trait used to translate a stable construct to its rustc counterpart.
+///
+/// This is basically a mirror of [crate::rustc_smir::Stable].
+pub trait RustcInternal<'tcx> {
+    type T;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T;
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/builder.rs b/compiler/rustc_smir/src/rustc_smir/builder.rs
new file mode 100644
index 000000000..8ff3958da
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/builder.rs
@@ -0,0 +1,55 @@
+//! Logic required to produce a monomorphic stable body.
+//!
+//! We first retrieve and monomorphize the rustc body representation, i.e., we generate a
+//! monomorphic body using internal representation.
+//! After that, we convert the internal representation into a stable one.
+use crate::rustc_smir::{Stable, Tables};
+use rustc_middle::mir;
+use rustc_middle::mir::visit::MutVisitor;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+
+/// Builds a monomorphic body for a given instance.
+pub struct BodyBuilder<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    instance: ty::Instance<'tcx>,
+}
+
+impl<'tcx> BodyBuilder<'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, instance: ty::Instance<'tcx>) -> Self {
+        BodyBuilder { tcx, instance }
+    }
+
+    pub fn build(mut self, tables: &mut Tables<'tcx>) -> stable_mir::mir::Body {
+        let mut body = self.tcx.instance_mir(self.instance.def).clone();
+        let generics = self.tcx.generics_of(self.instance.def_id());
+        if generics.requires_monomorphization(self.tcx) {
+            self.visit_body(&mut body);
+        }
+        body.stable(tables)
+    }
+
+    fn monomorphize<T>(&self, value: T) -> T
+    where
+        T: ty::TypeFoldable<TyCtxt<'tcx>>,
+    {
+        self.instance.instantiate_mir_and_normalize_erasing_regions(
+            self.tcx,
+            ty::ParamEnv::reveal_all(),
+            ty::EarlyBinder::bind(value),
+        )
+    }
+}
+
+impl<'tcx> MutVisitor<'tcx> for BodyBuilder<'tcx> {
+    fn visit_ty_const(&mut self, ct: &mut ty::Const<'tcx>, _location: mir::Location) {
+        *ct = self.monomorphize(*ct);
+    }
+
+    fn visit_ty(&mut self, ty: &mut Ty<'tcx>, _: mir::visit::TyContext) {
+        *ty = self.monomorphize(*ty);
+    }
+
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 6e83b9601..27596c08f 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -7,200 +7,269 @@
 //!
 //! For now, we are developing everything inside `rustc`, thus, we keep this module private.
 
+use crate::rustc_internal::{IndexMap, RustcInternal};
 use crate::rustc_smir::hir::def::DefKind;
 use crate::rustc_smir::stable_mir::ty::{BoundRegion, EarlyBoundRegion, Region};
 use rustc_hir as hir;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::{alloc_range, AllocId};
-use rustc_middle::ty::{self, Ty, TyCtxt, Variance};
+use rustc_middle::mir::mono::MonoItem;
+use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt, Variance};
 use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
 use rustc_target::abi::FieldIdx;
-use stable_mir::mir::{CopyNonOverlapping, UserTypeProjection, VariantIdx};
-use stable_mir::ty::{FloatTy, GenericParamDef, IntTy, Movability, RigidTy, Span, TyKind, UintTy};
-use stable_mir::{self, opaque, Context};
+use stable_mir::mir::mono::InstanceDef;
+use stable_mir::mir::{Body, CopyNonOverlapping, Statement, UserTypeProjection, VariantIdx};
+use stable_mir::ty::{
+    Const, ConstId, ConstantKind, FloatTy, GenericParamDef, IntTy, LineInfo, Movability, RigidTy,
+    Span, TyKind, UintTy,
+};
+use stable_mir::{self, opaque, Context, Filename};
+use std::cell::RefCell;
 use tracing::debug;
 
 mod alloc;
+mod builder;
 
-impl<'tcx> Context for Tables<'tcx> {
+impl<'tcx> Context for TablesWrapper<'tcx> {
     fn local_crate(&self) -> stable_mir::Crate {
-        smir_crate(self.tcx, LOCAL_CRATE)
+        let tables = self.0.borrow();
+        smir_crate(tables.tcx, LOCAL_CRATE)
     }
 
     fn external_crates(&self) -> Vec<stable_mir::Crate> {
-        self.tcx.crates(()).iter().map(|crate_num| smir_crate(self.tcx, *crate_num)).collect()
+        let tables = self.0.borrow();
+        tables.tcx.crates(()).iter().map(|crate_num| smir_crate(tables.tcx, *crate_num)).collect()
     }
 
-    fn find_crate(&self, name: &str) -> Option<stable_mir::Crate> {
-        [LOCAL_CRATE].iter().chain(self.tcx.crates(()).iter()).find_map(|crate_num| {
-            let crate_name = self.tcx.crate_name(*crate_num).to_string();
-            (name == crate_name).then(|| smir_crate(self.tcx, *crate_num))
-        })
+    fn find_crates(&self, name: &str) -> Vec<stable_mir::Crate> {
+        let tables = self.0.borrow();
+        let crates: Vec<stable_mir::Crate> = [LOCAL_CRATE]
+            .iter()
+            .chain(tables.tcx.crates(()).iter())
+            .map(|crate_num| {
+                let crate_name = tables.tcx.crate_name(*crate_num).to_string();
+                (name == crate_name).then(|| smir_crate(tables.tcx, *crate_num))
+            })
+            .into_iter()
+            .filter_map(|c| c)
+            .collect();
+        crates
     }
 
     fn name_of_def_id(&self, def_id: stable_mir::DefId) -> String {
-        self.tcx.def_path_str(self[def_id])
+        let tables = self.0.borrow();
+        tables.tcx.def_path_str(tables[def_id])
     }
 
-    fn print_span(&self, span: stable_mir::ty::Span) -> String {
-        self.tcx.sess.source_map().span_to_diagnostic_string(self[span])
+    fn span_to_string(&self, span: stable_mir::ty::Span) -> String {
+        let tables = self.0.borrow();
+        tables.tcx.sess.source_map().span_to_diagnostic_string(tables[span])
     }
 
-    fn def_kind(&mut self, def_id: stable_mir::DefId) -> stable_mir::DefKind {
-        self.tcx.def_kind(self[def_id]).stable(self)
+    fn get_filename(&self, span: &Span) -> Filename {
+        let tables = self.0.borrow();
+        opaque(
+            &tables
+                .tcx
+                .sess
+                .source_map()
+                .span_to_filename(tables[*span])
+                .display(rustc_span::FileNameDisplayPreference::Local)
+                .to_string(),
+        )
     }
 
-    fn span_of_an_item(&mut self, def_id: stable_mir::DefId) -> Span {
-        self.tcx.def_span(self[def_id]).stable(self)
+    fn get_lines(&self, span: &Span) -> LineInfo {
+        let tables = self.0.borrow();
+        let lines = &tables.tcx.sess.source_map().span_to_location_info(tables[*span]);
+        LineInfo { start_line: lines.1, start_col: lines.2, end_line: lines.3, end_col: lines.4 }
     }
 
-    fn all_local_items(&mut self) -> stable_mir::CrateItems {
-        self.tcx.mir_keys(()).iter().map(|item| self.crate_item(item.to_def_id())).collect()
+    fn def_kind(&self, def_id: stable_mir::DefId) -> stable_mir::DefKind {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.def_kind(tables[def_id]).stable(&mut *tables)
     }
 
-    fn entry_fn(&mut self) -> Option<stable_mir::CrateItem> {
-        Some(self.crate_item(self.tcx.entry_fn(())?.0))
+    fn span_of_an_item(&self, def_id: stable_mir::DefId) -> Span {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.def_span(tables[def_id]).stable(&mut *tables)
     }
 
-    fn all_trait_decls(&mut self) -> stable_mir::TraitDecls {
-        self.tcx
+    fn all_local_items(&self) -> stable_mir::CrateItems {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.mir_keys(()).iter().map(|item| tables.crate_item(item.to_def_id())).collect()
+    }
+
+    fn entry_fn(&self) -> Option<stable_mir::CrateItem> {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        Some(tables.crate_item(tcx.entry_fn(())?.0))
+    }
+
+    fn all_trait_decls(&self) -> stable_mir::TraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
             .traits(LOCAL_CRATE)
             .iter()
-            .map(|trait_def_id| self.trait_def(*trait_def_id))
+            .map(|trait_def_id| tables.trait_def(*trait_def_id))
             .collect()
     }
 
-    fn trait_decl(&mut self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
-        let def_id = self[trait_def.0];
-        let trait_def = self.tcx.trait_def(def_id);
-        trait_def.stable(self)
+    fn trait_decl(&self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[trait_def.0];
+        let trait_def = tables.tcx.trait_def(def_id);
+        trait_def.stable(&mut *tables)
     }
 
-    fn all_trait_impls(&mut self) -> stable_mir::ImplTraitDecls {
-        self.tcx
+    fn all_trait_impls(&self) -> stable_mir::ImplTraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
             .trait_impls_in_crate(LOCAL_CRATE)
             .iter()
-            .map(|impl_def_id| self.impl_def(*impl_def_id))
+            .map(|impl_def_id| tables.impl_def(*impl_def_id))
             .collect()
     }
 
-    fn trait_impl(&mut self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
-        let def_id = self[impl_def.0];
-        let impl_trait = self.tcx.impl_trait_ref(def_id).unwrap();
-        impl_trait.stable(self)
-    }
-
-    fn mir_body(&mut self, item: stable_mir::DefId) -> stable_mir::mir::Body {
-        let def_id = self[item];
-        let mir = self.tcx.instance_mir(ty::InstanceDef::Item(def_id));
-        stable_mir::mir::Body {
-            blocks: mir
-                .basic_blocks
-                .iter()
-                .map(|block| stable_mir::mir::BasicBlock {
-                    terminator: block.terminator().stable(self),
-                    statements: block
-                        .statements
-                        .iter()
-                        .map(|statement| statement.stable(self))
-                        .collect(),
-                })
-                .collect(),
-            locals: mir.local_decls.iter().map(|decl| self.intern_ty(decl.ty)).collect(),
-        }
+    fn trait_impl(&self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[impl_def.0];
+        let impl_trait = tables.tcx.impl_trait_ref(def_id).unwrap();
+        impl_trait.stable(&mut *tables)
     }
 
-    fn ty_kind(&mut self, ty: stable_mir::ty::Ty) -> TyKind {
-        self.types[ty.0].clone().stable(self)
+    fn mir_body(&self, item: stable_mir::DefId) -> stable_mir::mir::Body {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[item];
+        tables.tcx.instance_mir(ty::InstanceDef::Item(def_id)).stable(&mut tables)
     }
 
-    fn mk_ty(&mut self, kind: TyKind) -> stable_mir::ty::Ty {
-        let n = self.types.len();
-        self.types.push(MaybeStable::Stable(kind));
-        stable_mir::ty::Ty(n)
+    fn ty_kind(&self, ty: stable_mir::ty::Ty) -> TyKind {
+        let mut tables = self.0.borrow_mut();
+        tables.types[ty].kind().stable(&mut *tables)
     }
 
-    fn generics_of(&mut self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
-        let def_id = self[def_id];
-        let generics = self.tcx.generics_of(def_id);
-        generics.stable(self)
+    fn generics_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        generics.stable(&mut *tables)
     }
 
-    fn predicates_of(&mut self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
-        let def_id = self[def_id];
-        let ty::GenericPredicates { parent, predicates } = self.tcx.predicates_of(def_id);
+    fn predicates_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let ty::GenericPredicates { parent, predicates } = tables.tcx.predicates_of(def_id);
         stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| self.trait_def(did)),
+            parent: parent.map(|did| tables.trait_def(did)),
             predicates: predicates
                 .iter()
                 .map(|(clause, span)| {
-                    (clause.as_predicate().kind().skip_binder().stable(self), span.stable(self))
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
                 })
                 .collect(),
         }
     }
 
     fn explicit_predicates_of(
-        &mut self,
+        &self,
         def_id: stable_mir::DefId,
     ) -> stable_mir::ty::GenericPredicates {
-        let def_id = self[def_id];
-        let ty::GenericPredicates { parent, predicates } = self.tcx.explicit_predicates_of(def_id);
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let ty::GenericPredicates { parent, predicates } =
+            tables.tcx.explicit_predicates_of(def_id);
         stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| self.trait_def(did)),
+            parent: parent.map(|did| tables.trait_def(did)),
             predicates: predicates
                 .iter()
                 .map(|(clause, span)| {
-                    (clause.as_predicate().kind().skip_binder().stable(self), span.stable(self))
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
                 })
                 .collect(),
         }
     }
-}
 
-#[derive(Clone)]
-pub enum MaybeStable<S, R> {
-    Stable(S),
-    Rustc(R),
-}
+    fn instance_body(&self, def: InstanceDef) -> Body {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        builder::BodyBuilder::new(tables.tcx, instance).build(&mut *tables)
+    }
 
-impl<'tcx, S, R> MaybeStable<S, R> {
-    fn stable(self, tables: &mut Tables<'tcx>) -> S
-    where
-        R: Stable<'tcx, T = S>,
-    {
-        match self {
-            MaybeStable::Stable(s) => s,
-            MaybeStable::Rustc(r) => r.stable(tables),
-        }
+    fn instance_ty(&self, def: InstanceDef) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        instance.ty(tables.tcx, ParamEnv::empty()).stable(&mut *tables)
     }
-}
 
-impl<S, R: PartialEq> PartialEq<R> for MaybeStable<S, R> {
-    fn eq(&self, other: &R) -> bool {
-        match self {
-            MaybeStable::Stable(_) => false,
-            MaybeStable::Rustc(r) => r == other,
+    fn instance_def_id(&self, def: InstanceDef) -> stable_mir::DefId {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables.instances[def].def_id();
+        tables.create_def_id(def_id)
+    }
+
+    fn instance_mangled_name(&self, def: InstanceDef) -> String {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        tables.tcx.symbol_name(instance).name.to_string()
+    }
+
+    fn mono_instance(&self, item: stable_mir::CrateItem) -> stable_mir::mir::mono::Instance {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[item.0];
+        Instance::mono(tables.tcx, def_id).stable(&mut *tables)
+    }
+
+    fn requires_monomorphization(&self, def_id: stable_mir::DefId) -> bool {
+        let tables = self.0.borrow();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        let result = generics.requires_monomorphization(tables.tcx);
+        result
+    }
+
+    fn resolve_instance(
+        &self,
+        def: stable_mir::ty::FnDef,
+        args: &stable_mir::ty::GenericArgs,
+    ) -> Option<stable_mir::mir::mono::Instance> {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let args_ref = args.internal(&mut *tables);
+        match Instance::resolve(tables.tcx, ParamEnv::reveal_all(), def_id, args_ref) {
+            Ok(Some(instance)) => Some(instance.stable(&mut *tables)),
+            Ok(None) | Err(_) => None,
         }
     }
 }
 
+pub(crate) struct TablesWrapper<'tcx>(pub(crate) RefCell<Tables<'tcx>>);
+
 pub struct Tables<'tcx> {
-    pub tcx: TyCtxt<'tcx>,
-    pub def_ids: Vec<DefId>,
-    pub alloc_ids: Vec<AllocId>,
-    pub spans: Vec<rustc_span::Span>,
-    pub types: Vec<MaybeStable<stable_mir::ty::TyKind, Ty<'tcx>>>,
+    pub(crate) tcx: TyCtxt<'tcx>,
+    pub(crate) def_ids: IndexMap<DefId, stable_mir::DefId>,
+    pub(crate) alloc_ids: IndexMap<AllocId, stable_mir::AllocId>,
+    pub(crate) spans: IndexMap<rustc_span::Span, Span>,
+    pub(crate) types: IndexMap<Ty<'tcx>, stable_mir::ty::Ty>,
+    pub(crate) instances: IndexMap<ty::Instance<'tcx>, InstanceDef>,
+    pub(crate) constants: IndexMap<mir::Const<'tcx>, ConstId>,
 }
 
 impl<'tcx> Tables<'tcx> {
     fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty {
-        if let Some(id) = self.types.iter().position(|t| *t == ty) {
-            return stable_mir::ty::Ty(id);
-        }
-        let id = self.types.len();
-        self.types.push(MaybeStable::Rustc(ty));
-        stable_mir::ty::Ty(id)
+        self.types.create_or_fetch(ty)
+    }
+
+    fn intern_const(&mut self, constant: mir::Const<'tcx>) -> ConstId {
+        self.constants.create_or_fetch(constant)
     }
 }
 
@@ -213,51 +282,102 @@ fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
 }
 
 /// Trait used to convert between an internal MIR type to a Stable MIR type.
-pub(crate) trait Stable<'tcx> {
+pub trait Stable<'tcx> {
     /// The stable representation of the type implementing Stable.
     type T;
     /// Converts an object to the equivalent Stable MIR representation.
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T;
 }
 
+impl<'tcx> Stable<'tcx> for mir::Body<'tcx> {
+    type T = stable_mir::mir::Body;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::Body::new(
+            self.basic_blocks
+                .iter()
+                .map(|block| stable_mir::mir::BasicBlock {
+                    terminator: block.terminator().stable(tables),
+                    statements: block
+                        .statements
+                        .iter()
+                        .map(|statement| statement.stable(tables))
+                        .collect(),
+                })
+                .collect(),
+            self.local_decls
+                .iter()
+                .map(|decl| stable_mir::mir::LocalDecl {
+                    ty: decl.ty.stable(tables),
+                    span: decl.source_info.span.stable(tables),
+                })
+                .collect(),
+            self.arg_count,
+        )
+    }
+}
+
 impl<'tcx> Stable<'tcx> for mir::Statement<'tcx> {
     type T = stable_mir::mir::Statement;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::StatementKind::*;
-        match &self.kind {
-            Assign(assign) => {
-                stable_mir::mir::Statement::Assign(assign.0.stable(tables), assign.1.stable(tables))
-            }
-            FakeRead(fake_read_place) => stable_mir::mir::Statement::FakeRead(
-                fake_read_place.0.stable(tables),
-                fake_read_place.1.stable(tables),
+        Statement { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::StatementKind<'tcx> {
+    type T = stable_mir::mir::StatementKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            mir::StatementKind::Assign(assign) => stable_mir::mir::StatementKind::Assign(
+                assign.0.stable(tables),
+                assign.1.stable(tables),
             ),
-            SetDiscriminant { place: plc, variant_index: idx } => {
-                stable_mir::mir::Statement::SetDiscriminant {
-                    place: plc.as_ref().stable(tables),
-                    variant_index: idx.stable(tables),
+            mir::StatementKind::FakeRead(fake_read_place) => {
+                stable_mir::mir::StatementKind::FakeRead(
+                    fake_read_place.0.stable(tables),
+                    fake_read_place.1.stable(tables),
+                )
+            }
+            mir::StatementKind::SetDiscriminant { place, variant_index } => {
+                stable_mir::mir::StatementKind::SetDiscriminant {
+                    place: place.as_ref().stable(tables),
+                    variant_index: variant_index.stable(tables),
                 }
             }
-            Deinit(place) => stable_mir::mir::Statement::Deinit(place.stable(tables)),
-            StorageLive(place) => stable_mir::mir::Statement::StorageLive(place.stable(tables)),
-            StorageDead(place) => stable_mir::mir::Statement::StorageDead(place.stable(tables)),
-            Retag(retag, place) => {
-                stable_mir::mir::Statement::Retag(retag.stable(tables), place.stable(tables))
+            mir::StatementKind::Deinit(place) => {
+                stable_mir::mir::StatementKind::Deinit(place.stable(tables))
+            }
+
+            mir::StatementKind::StorageLive(place) => {
+                stable_mir::mir::StatementKind::StorageLive(place.stable(tables))
+            }
+
+            mir::StatementKind::StorageDead(place) => {
+                stable_mir::mir::StatementKind::StorageDead(place.stable(tables))
+            }
+            mir::StatementKind::Retag(retag, place) => {
+                stable_mir::mir::StatementKind::Retag(retag.stable(tables), place.stable(tables))
             }
-            PlaceMention(place) => stable_mir::mir::Statement::PlaceMention(place.stable(tables)),
-            AscribeUserType(place_projection, variance) => {
-                stable_mir::mir::Statement::AscribeUserType {
+            mir::StatementKind::PlaceMention(place) => {
+                stable_mir::mir::StatementKind::PlaceMention(place.stable(tables))
+            }
+            mir::StatementKind::AscribeUserType(place_projection, variance) => {
+                stable_mir::mir::StatementKind::AscribeUserType {
                     place: place_projection.as_ref().0.stable(tables),
                     projections: place_projection.as_ref().1.stable(tables),
                     variance: variance.stable(tables),
                 }
             }
-            Coverage(coverage) => stable_mir::mir::Statement::Coverage(opaque(coverage)),
-            Intrinsic(intrinstic) => {
-                stable_mir::mir::Statement::Intrinsic(intrinstic.stable(tables))
+            mir::StatementKind::Coverage(coverage) => {
+                stable_mir::mir::StatementKind::Coverage(opaque(coverage))
+            }
+            mir::StatementKind::Intrinsic(intrinstic) => {
+                stable_mir::mir::StatementKind::Intrinsic(intrinstic.stable(tables))
             }
-            ConstEvalCounter => stable_mir::mir::Statement::ConstEvalCounter,
-            Nop => stable_mir::mir::Statement::Nop,
+            mir::StatementKind::ConstEvalCounter => {
+                stable_mir::mir::StatementKind::ConstEvalCounter
+            }
+            mir::StatementKind::Nop => stable_mir::mir::StatementKind::Nop,
         }
     }
 }
@@ -287,7 +407,7 @@ impl<'tcx> Stable<'tcx> for mir::Rvalue<'tcx> {
             Cast(cast_kind, op, ty) => stable_mir::mir::Rvalue::Cast(
                 cast_kind.stable(tables),
                 op.stable(tables),
-                tables.intern_ty(*ty),
+                ty.stable(tables),
             ),
             BinaryOp(bin_op, ops) => stable_mir::mir::Rvalue::BinaryOp(
                 bin_op.stable(tables),
@@ -300,7 +420,7 @@ impl<'tcx> Stable<'tcx> for mir::Rvalue<'tcx> {
                 ops.1.stable(tables),
             ),
             NullaryOp(null_op, ty) => {
-                stable_mir::mir::Rvalue::NullaryOp(null_op.stable(tables), tables.intern_ty(*ty))
+                stable_mir::mir::Rvalue::NullaryOp(null_op.stable(tables), ty.stable(tables))
             }
             UnaryOp(un_op, op) => {
                 stable_mir::mir::Rvalue::UnaryOp(un_op.stable(tables), op.stable(tables))
@@ -311,7 +431,7 @@ impl<'tcx> Stable<'tcx> for mir::Rvalue<'tcx> {
                 stable_mir::mir::Rvalue::Aggregate(agg_kind.stable(tables), operands)
             }
             ShallowInitBox(op, ty) => {
-                stable_mir::mir::Rvalue::ShallowInitBox(op.stable(tables), tables.intern_ty(*ty))
+                stable_mir::mir::Rvalue::ShallowInitBox(op.stable(tables), ty.stable(tables))
             }
             CopyForDeref(place) => stable_mir::mir::Rvalue::CopyForDeref(place.stable(tables)),
         }
@@ -455,7 +575,7 @@ impl<'tcx> Stable<'tcx> for ty::TermKind<'tcx> {
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
         use stable_mir::ty::TermKind;
         match self {
-            ty::TermKind::Ty(ty) => TermKind::Type(tables.intern_ty(*ty)),
+            ty::TermKind::Ty(ty) => TermKind::Type(ty.stable(tables)),
             ty::TermKind::Const(cnst) => {
                 let cnst = cnst.stable(tables);
                 TermKind::Const(cnst)
@@ -529,6 +649,13 @@ impl<'tcx> Stable<'tcx> for FieldIdx {
     }
 }
 
+impl<'tcx> Stable<'tcx> for (rustc_target::abi::VariantIdx, FieldIdx) {
+    type T = (usize, usize);
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        (self.0.as_usize(), self.1.as_usize())
+    }
+}
+
 impl<'tcx> Stable<'tcx> for mir::Operand<'tcx> {
     type T = stable_mir::mir::Operand;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
@@ -673,11 +800,11 @@ impl<'tcx> Stable<'tcx> for mir::AssertMessage<'tcx> {
             AssertKind::RemainderByZero(op) => {
                 stable_mir::mir::AssertMessage::RemainderByZero(op.stable(tables))
             }
-            AssertKind::ResumedAfterReturn(generator) => {
-                stable_mir::mir::AssertMessage::ResumedAfterReturn(generator.stable(tables))
+            AssertKind::ResumedAfterReturn(coroutine) => {
+                stable_mir::mir::AssertMessage::ResumedAfterReturn(coroutine.stable(tables))
             }
-            AssertKind::ResumedAfterPanic(generator) => {
-                stable_mir::mir::AssertMessage::ResumedAfterPanic(generator.stable(tables))
+            AssertKind::ResumedAfterPanic(coroutine) => {
+                stable_mir::mir::AssertMessage::ResumedAfterPanic(coroutine.stable(tables))
             }
             AssertKind::MisalignedPointerDereference { required, found } => {
                 stable_mir::mir::AssertMessage::MisalignedPointerDereference {
@@ -736,7 +863,7 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
         match self {
             mir::AggregateKind::Array(ty) => {
-                stable_mir::mir::AggregateKind::Array(tables.intern_ty(*ty))
+                stable_mir::mir::AggregateKind::Array(ty.stable(tables))
             }
             mir::AggregateKind::Tuple => stable_mir::mir::AggregateKind::Tuple,
             mir::AggregateKind::Adt(def_id, var_idx, generic_arg, user_ty_index, field_idx) => {
@@ -754,9 +881,9 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
                     generic_arg.stable(tables),
                 )
             }
-            mir::AggregateKind::Generator(def_id, generic_arg, movability) => {
-                stable_mir::mir::AggregateKind::Generator(
-                    tables.generator_def(*def_id),
+            mir::AggregateKind::Coroutine(def_id, generic_arg, movability) => {
+                stable_mir::mir::AggregateKind::Coroutine(
+                    tables.coroutine_def(*def_id),
                     generic_arg.stable(tables),
                     movability.stable(tables),
                 )
@@ -765,20 +892,30 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
     }
 }
 
-impl<'tcx> Stable<'tcx> for rustc_hir::GeneratorKind {
-    type T = stable_mir::mir::GeneratorKind;
+impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineSource {
+    type T = stable_mir::mir::CoroutineSource;
     fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use rustc_hir::{AsyncGeneratorKind, GeneratorKind};
+        use rustc_hir::CoroutineSource;
+        match self {
+            CoroutineSource::Block => stable_mir::mir::CoroutineSource::Block,
+            CoroutineSource::Closure => stable_mir::mir::CoroutineSource::Closure,
+            CoroutineSource::Fn => stable_mir::mir::CoroutineSource::Fn,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineKind {
+    type T = stable_mir::mir::CoroutineKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::CoroutineKind;
         match self {
-            GeneratorKind::Async(async_gen) => {
-                let async_gen = match async_gen {
-                    AsyncGeneratorKind::Block => stable_mir::mir::AsyncGeneratorKind::Block,
-                    AsyncGeneratorKind::Closure => stable_mir::mir::AsyncGeneratorKind::Closure,
-                    AsyncGeneratorKind::Fn => stable_mir::mir::AsyncGeneratorKind::Fn,
-                };
-                stable_mir::mir::GeneratorKind::Async(async_gen)
-            }
-            GeneratorKind::Gen => stable_mir::mir::GeneratorKind::Gen,
+            CoroutineKind::Async(source) => {
+                stable_mir::mir::CoroutineKind::Async(source.stable(tables))
+            }
+            CoroutineKind::Gen(source) => {
+                stable_mir::mir::CoroutineKind::Gen(source.stable(tables))
+            }
+            CoroutineKind::Coroutine => stable_mir::mir::CoroutineKind::Coroutine,
         }
     }
 }
@@ -806,11 +943,20 @@ impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> {
 impl<'tcx> Stable<'tcx> for mir::Terminator<'tcx> {
     type T = stable_mir::mir::Terminator;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::TerminatorKind::*;
         use stable_mir::mir::Terminator;
-        match &self.kind {
-            Goto { target } => Terminator::Goto { target: target.as_usize() },
-            SwitchInt { discr, targets } => Terminator::SwitchInt {
+        Terminator { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> {
+    type T = stable_mir::mir::TerminatorKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::mir::TerminatorKind;
+        match self {
+            mir::TerminatorKind::Goto { target } => {
+                TerminatorKind::Goto { target: target.as_usize() }
+            }
+            mir::TerminatorKind::SwitchInt { discr, targets } => TerminatorKind::SwitchInt {
                 discr: discr.stable(tables),
                 targets: targets
                     .iter()
@@ -821,42 +967,60 @@ impl<'tcx> Stable<'tcx> for mir::Terminator<'tcx> {
                     .collect(),
                 otherwise: targets.otherwise().as_usize(),
             },
-            UnwindResume => Terminator::Resume,
-            UnwindTerminate(_) => Terminator::Abort,
-            Return => Terminator::Return,
-            Unreachable => Terminator::Unreachable,
-            Drop { place, target, unwind, replace: _ } => Terminator::Drop {
-                place: place.stable(tables),
-                target: target.as_usize(),
-                unwind: unwind.stable(tables),
-            },
-            Call { func, args, destination, target, unwind, call_source: _, fn_span: _ } => {
-                Terminator::Call {
-                    func: func.stable(tables),
-                    args: args.iter().map(|arg| arg.stable(tables)).collect(),
-                    destination: destination.stable(tables),
-                    target: target.map(|t| t.as_usize()),
+            mir::TerminatorKind::UnwindResume => TerminatorKind::Resume,
+            mir::TerminatorKind::UnwindTerminate(_) => TerminatorKind::Abort,
+            mir::TerminatorKind::Return => TerminatorKind::Return,
+            mir::TerminatorKind::Unreachable => TerminatorKind::Unreachable,
+            mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => {
+                TerminatorKind::Drop {
+                    place: place.stable(tables),
+                    target: target.as_usize(),
                     unwind: unwind.stable(tables),
                 }
             }
-            Assert { cond, expected, msg, target, unwind } => Terminator::Assert {
-                cond: cond.stable(tables),
-                expected: *expected,
-                msg: msg.stable(tables),
-                target: target.as_usize(),
+            mir::TerminatorKind::Call {
+                func,
+                args,
+                destination,
+                target,
+                unwind,
+                call_source: _,
+                fn_span: _,
+            } => TerminatorKind::Call {
+                func: func.stable(tables),
+                args: args.iter().map(|arg| arg.stable(tables)).collect(),
+                destination: destination.stable(tables),
+                target: target.map(|t| t.as_usize()),
                 unwind: unwind.stable(tables),
             },
-            InlineAsm { template, operands, options, line_spans, destination, unwind } => {
-                Terminator::InlineAsm {
-                    template: format!("{template:?}"),
-                    operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
-                    options: format!("{options:?}"),
-                    line_spans: format!("{line_spans:?}"),
-                    destination: destination.map(|d| d.as_usize()),
+            mir::TerminatorKind::Assert { cond, expected, msg, target, unwind } => {
+                TerminatorKind::Assert {
+                    cond: cond.stable(tables),
+                    expected: *expected,
+                    msg: msg.stable(tables),
+                    target: target.as_usize(),
                     unwind: unwind.stable(tables),
                 }
             }
-            Yield { .. } | GeneratorDrop | FalseEdge { .. } | FalseUnwind { .. } => unreachable!(),
+            mir::TerminatorKind::InlineAsm {
+                template,
+                operands,
+                options,
+                line_spans,
+                destination,
+                unwind,
+            } => TerminatorKind::InlineAsm {
+                template: format!("{template:?}"),
+                operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
+                options: format!("{options:?}"),
+                line_spans: format!("{line_spans:?}"),
+                destination: destination.map(|d| d.as_usize()),
+                unwind: unwind.stable(tables),
+            },
+            mir::TerminatorKind::Yield { .. }
+            | mir::TerminatorKind::CoroutineDrop
+            | mir::TerminatorKind::FalseEdge { .. }
+            | mir::TerminatorKind::FalseUnwind { .. } => unreachable!(),
         }
     }
 }
@@ -877,7 +1041,7 @@ impl<'tcx> Stable<'tcx> for ty::GenericArgKind<'tcx> {
         use stable_mir::ty::GenericArgKind;
         match self {
             ty::GenericArgKind::Lifetime(region) => GenericArgKind::Lifetime(region.stable(tables)),
-            ty::GenericArgKind::Type(ty) => GenericArgKind::Type(tables.intern_ty(*ty)),
+            ty::GenericArgKind::Type(ty) => GenericArgKind::Type(ty.stable(tables)),
             ty::GenericArgKind::Const(cnst) => GenericArgKind::Const(cnst.stable(tables)),
         }
     }
@@ -923,11 +1087,7 @@ impl<'tcx> Stable<'tcx> for ty::FnSig<'tcx> {
         use stable_mir::ty::{Abi, FnSig};
 
         FnSig {
-            inputs_and_output: self
-                .inputs_and_output
-                .iter()
-                .map(|ty| tables.intern_ty(ty))
-                .collect(),
+            inputs_and_output: self.inputs_and_output.iter().map(|ty| ty.stable(tables)).collect(),
             c_variadic: self.c_variadic,
             unsafety: self.unsafety.stable(tables),
             abi: match self.abi {
@@ -1065,9 +1225,16 @@ impl<'tcx> Stable<'tcx> for hir::Movability {
 }
 
 impl<'tcx> Stable<'tcx> for Ty<'tcx> {
+    type T = stable_mir::ty::Ty;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.intern_ty(*self)
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TyKind<'tcx> {
     type T = stable_mir::ty::TyKind;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        match self.kind() {
+        match self {
             ty::Bool => TyKind::RigidTy(RigidTy::Bool),
             ty::Char => TyKind::RigidTy(RigidTy::Char),
             ty::Int(int_ty) => TyKind::RigidTy(RigidTy::Int(int_ty.stable(tables))),
@@ -1080,15 +1247,15 @@ impl<'tcx> Stable<'tcx> for Ty<'tcx> {
             ty::Foreign(def_id) => TyKind::RigidTy(RigidTy::Foreign(tables.foreign_def(*def_id))),
             ty::Str => TyKind::RigidTy(RigidTy::Str),
             ty::Array(ty, constant) => {
-                TyKind::RigidTy(RigidTy::Array(tables.intern_ty(*ty), constant.stable(tables)))
+                TyKind::RigidTy(RigidTy::Array(ty.stable(tables), constant.stable(tables)))
             }
-            ty::Slice(ty) => TyKind::RigidTy(RigidTy::Slice(tables.intern_ty(*ty))),
+            ty::Slice(ty) => TyKind::RigidTy(RigidTy::Slice(ty.stable(tables))),
             ty::RawPtr(ty::TypeAndMut { ty, mutbl }) => {
-                TyKind::RigidTy(RigidTy::RawPtr(tables.intern_ty(*ty), mutbl.stable(tables)))
+                TyKind::RigidTy(RigidTy::RawPtr(ty.stable(tables), mutbl.stable(tables)))
             }
             ty::Ref(region, ty, mutbl) => TyKind::RigidTy(RigidTy::Ref(
                 region.stable(tables),
-                tables.intern_ty(*ty),
+                ty.stable(tables),
                 mutbl.stable(tables),
             )),
             ty::FnDef(def_id, generic_args) => {
@@ -1109,15 +1276,15 @@ impl<'tcx> Stable<'tcx> for Ty<'tcx> {
                 tables.closure_def(*def_id),
                 generic_args.stable(tables),
             )),
-            ty::Generator(def_id, generic_args, movability) => TyKind::RigidTy(RigidTy::Generator(
-                tables.generator_def(*def_id),
+            ty::Coroutine(def_id, generic_args, movability) => TyKind::RigidTy(RigidTy::Coroutine(
+                tables.coroutine_def(*def_id),
                 generic_args.stable(tables),
                 movability.stable(tables),
             )),
             ty::Never => TyKind::RigidTy(RigidTy::Never),
-            ty::Tuple(fields) => TyKind::RigidTy(RigidTy::Tuple(
-                fields.iter().map(|ty| tables.intern_ty(ty)).collect(),
-            )),
+            ty::Tuple(fields) => {
+                TyKind::RigidTy(RigidTy::Tuple(fields.iter().map(|ty| ty.stable(tables)).collect()))
+            }
             ty::Alias(alias_kind, alias_ty) => {
                 TyKind::Alias(alias_kind.stable(tables), alias_ty.stable(tables))
             }
@@ -1125,7 +1292,7 @@ impl<'tcx> Stable<'tcx> for Ty<'tcx> {
             ty::Bound(debruijn_idx, bound_ty) => {
                 TyKind::Bound(debruijn_idx.as_usize(), bound_ty.stable(tables))
             }
-            ty::Placeholder(..) | ty::GeneratorWitness(..) | ty::Infer(_) | ty::Error(_) => {
+            ty::Placeholder(..) | ty::CoroutineWitness(..) | ty::Infer(_) | ty::Error(_) => {
                 unreachable!();
             }
         }
@@ -1136,32 +1303,36 @@ impl<'tcx> Stable<'tcx> for ty::Const<'tcx> {
     type T = stable_mir::ty::Const;
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        stable_mir::ty::Const {
-            literal: match self.kind() {
-                ty::Value(val) => {
-                    let const_val = tables.tcx.valtree_to_const_val((self.ty(), val));
+        let kind = match self.kind() {
+            ty::Value(val) => {
+                let const_val = tables.tcx.valtree_to_const_val((self.ty(), val));
+                if matches!(const_val, mir::ConstValue::ZeroSized) {
+                    ConstantKind::ZeroSized
+                } else {
                     stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation(
                         self.ty(),
                         const_val,
                         tables,
                     ))
                 }
-                ty::ParamCt(param) => stable_mir::ty::ConstantKind::Param(param.stable(tables)),
-                ty::ErrorCt(_) => unreachable!(),
-                ty::InferCt(_) => unreachable!(),
-                ty::BoundCt(_, _) => unimplemented!(),
-                ty::PlaceholderCt(_) => unimplemented!(),
-                ty::Unevaluated(uv) => {
-                    stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
-                        def: tables.const_def(uv.def),
-                        args: uv.args.stable(tables),
-                        promoted: None,
-                    })
-                }
-                ty::ExprCt(_) => unimplemented!(),
-            },
-            ty: tables.intern_ty(self.ty()),
-        }
+            }
+            ty::ParamCt(param) => stable_mir::ty::ConstantKind::Param(param.stable(tables)),
+            ty::ErrorCt(_) => unreachable!(),
+            ty::InferCt(_) => unreachable!(),
+            ty::BoundCt(_, _) => unimplemented!(),
+            ty::PlaceholderCt(_) => unimplemented!(),
+            ty::Unevaluated(uv) => {
+                stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
+                    def: tables.const_def(uv.def),
+                    args: uv.args.stable(tables),
+                    promoted: None,
+                })
+            }
+            ty::ExprCt(_) => unimplemented!(),
+        };
+        let ty = self.ty().stable(tables);
+        let id = tables.intern_const(mir::Const::Ty(*self));
+        Const::new(kind, ty, id)
     }
 }
 
@@ -1246,22 +1417,28 @@ impl<'tcx> Stable<'tcx> for rustc_middle::mir::Const<'tcx> {
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
         match *self {
             mir::Const::Ty(c) => c.stable(tables),
-            mir::Const::Unevaluated(unev_const, ty) => stable_mir::ty::Const {
-                literal: stable_mir::ty::ConstantKind::Unevaluated(
-                    stable_mir::ty::UnevaluatedConst {
+            mir::Const::Unevaluated(unev_const, ty) => {
+                let kind =
+                    stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
                         def: tables.const_def(unev_const.def),
                         args: unev_const.args.stable(tables),
                         promoted: unev_const.promoted.map(|u| u.as_u32()),
-                    },
-                ),
-                ty: tables.intern_ty(ty),
-            },
-            mir::Const::Val(val, ty) => stable_mir::ty::Const {
-                literal: stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation(
-                    ty, val, tables,
-                )),
-                ty: tables.intern_ty(ty),
-            },
+                    });
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(kind, ty, id)
+            }
+            mir::Const::Val(val, ty) if matches!(val, mir::ConstValue::ZeroSized) => {
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(ConstantKind::ZeroSized, ty, id)
+            }
+            mir::Const::Val(val, ty) => {
+                let kind = ConstantKind::Allocated(alloc::new_allocation(ty, val, tables));
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(kind, ty, id)
+            }
         }
     }
 }
@@ -1375,30 +1552,32 @@ impl<'tcx> Stable<'tcx> for ty::ClauseKind<'tcx> {
     type T = stable_mir::ty::ClauseKind;
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use ty::ClauseKind::*;
+        use ty::ClauseKind;
         match *self {
-            Trait(trait_object) => stable_mir::ty::ClauseKind::Trait(trait_object.stable(tables)),
-            RegionOutlives(region_outlives) => {
+            ClauseKind::Trait(trait_object) => {
+                stable_mir::ty::ClauseKind::Trait(trait_object.stable(tables))
+            }
+            ClauseKind::RegionOutlives(region_outlives) => {
                 stable_mir::ty::ClauseKind::RegionOutlives(region_outlives.stable(tables))
             }
-            TypeOutlives(type_outlives) => {
+            ClauseKind::TypeOutlives(type_outlives) => {
                 let ty::OutlivesPredicate::<_, _>(a, b) = type_outlives;
                 stable_mir::ty::ClauseKind::TypeOutlives(stable_mir::ty::OutlivesPredicate(
-                    tables.intern_ty(a),
+                    a.stable(tables),
                     b.stable(tables),
                 ))
             }
-            Projection(projection_predicate) => {
+            ClauseKind::Projection(projection_predicate) => {
                 stable_mir::ty::ClauseKind::Projection(projection_predicate.stable(tables))
             }
-            ConstArgHasType(const_, ty) => stable_mir::ty::ClauseKind::ConstArgHasType(
+            ClauseKind::ConstArgHasType(const_, ty) => stable_mir::ty::ClauseKind::ConstArgHasType(
                 const_.stable(tables),
-                tables.intern_ty(ty),
+                ty.stable(tables),
             ),
-            WellFormed(generic_arg) => {
+            ClauseKind::WellFormed(generic_arg) => {
                 stable_mir::ty::ClauseKind::WellFormed(generic_arg.unpack().stable(tables))
             }
-            ConstEvaluatable(const_) => {
+            ClauseKind::ConstEvaluatable(const_) => {
                 stable_mir::ty::ClauseKind::ConstEvaluatable(const_.stable(tables))
             }
         }
@@ -1423,7 +1602,7 @@ impl<'tcx> Stable<'tcx> for ty::SubtypePredicate<'tcx> {
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
         let ty::SubtypePredicate { a, b, a_is_expected: _ } = self;
-        stable_mir::ty::SubtypePredicate { a: tables.intern_ty(*a), b: tables.intern_ty(*b) }
+        stable_mir::ty::SubtypePredicate { a: a.stable(tables), b: b.stable(tables) }
     }
 }
 
@@ -1432,7 +1611,7 @@ impl<'tcx> Stable<'tcx> for ty::CoercePredicate<'tcx> {
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
         let ty::CoercePredicate { a, b } = self;
-        stable_mir::ty::CoercePredicate { a: tables.intern_ty(*a), b: tables.intern_ty(*b) }
+        stable_mir::ty::CoercePredicate { a: a.stable(tables), b: b.stable(tables) }
     }
 }
 
@@ -1553,3 +1732,38 @@ impl<'tcx> Stable<'tcx> for DefKind {
         opaque(self)
     }
 }
+
+impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
+    type T = stable_mir::mir::mono::Instance;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let def = tables.instance_def(*self);
+        let kind = match self.def {
+            ty::InstanceDef::Item(..) => stable_mir::mir::mono::InstanceKind::Item,
+            ty::InstanceDef::Intrinsic(..) => stable_mir::mir::mono::InstanceKind::Intrinsic,
+            ty::InstanceDef::Virtual(..) => stable_mir::mir::mono::InstanceKind::Virtual,
+            ty::InstanceDef::VTableShim(..)
+            | ty::InstanceDef::ReifyShim(..)
+            | ty::InstanceDef::FnPtrAddrShim(..)
+            | ty::InstanceDef::ClosureOnceShim { .. }
+            | ty::InstanceDef::ThreadLocalShim(..)
+            | ty::InstanceDef::DropGlue(..)
+            | ty::InstanceDef::CloneShim(..)
+            | ty::InstanceDef::FnPtrShim(..) => stable_mir::mir::mono::InstanceKind::Shim,
+        };
+        stable_mir::mir::mono::Instance { def, kind }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for MonoItem<'tcx> {
+    type T = stable_mir::mir::mono::MonoItem;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::mir::mono::MonoItem as StableMonoItem;
+        match self {
+            MonoItem::Fn(instance) => StableMonoItem::Fn(instance.stable(tables)),
+            MonoItem::Static(def_id) => StableMonoItem::Static(tables.static_def(*def_id)),
+            MonoItem::GlobalAsm(item_id) => StableMonoItem::GlobalAsm(opaque(item_id)),
+        }
+    }
+}