summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_monomorphize
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_monomorphize')
-rw-r--r--compiler/rustc_monomorphize/src/collector.rs49
-rw-r--r--compiler/rustc_monomorphize/src/partitioning.rs228
-rw-r--r--compiler/rustc_monomorphize/src/polymorphize.rs28
-rw-r--r--compiler/rustc_monomorphize/src/util.rs4
4 files changed, 179 insertions, 130 deletions
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 242269e9d..55b14ce1c 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -178,11 +178,11 @@ use rustc_middle::mir::{self, Local, Location};
use rustc_middle::query::TyCtxtAt;
use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
use rustc_middle::ty::{
self, GenericParamDefKind, Instance, InstanceDef, Ty, TyCtxt, TypeFoldable, TypeVisitableExt,
VtblEntry,
};
+use rustc_middle::ty::{GenericArgKind, GenericArgs};
use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext};
use rustc_session::config::EntryFnType;
use rustc_session::lint::builtin::LARGE_ASSIGNMENTS;
@@ -384,7 +384,7 @@ fn collect_items_rec<'tcx>(
if let Ok(alloc) = tcx.eval_static_initializer(def_id) {
for &id in alloc.inner().provenance().ptrs().values() {
- collect_miri(tcx, id, &mut used_items);
+ collect_alloc(tcx, id, &mut used_items);
}
}
@@ -393,7 +393,7 @@ fn collect_items_rec<'tcx>(
starting_item.span,
MonoItem::Fn(Instance {
def: InstanceDef::ThreadLocalShim(def_id),
- substs: InternalSubsts::empty(),
+ args: GenericArgs::empty(),
}),
));
}
@@ -555,7 +555,7 @@ fn check_recursion_limit<'tcx>(
fn check_type_length_limit<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
let type_length = instance
- .substs
+ .args
.iter()
.flat_map(|arg| arg.walk())
.filter(|arg| match arg.unpack() {
@@ -659,11 +659,11 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
let source_ty = operand.ty(self.body, self.tcx);
let source_ty = self.monomorphize(source_ty);
match *source_ty.kind() {
- ty::Closure(def_id, substs) => {
+ ty::Closure(def_id, args) => {
let instance = Instance::resolve_closure(
self.tcx,
def_id,
- substs,
+ args,
ty::ClosureKind::FnOnce,
)
.expect("failed to normalize and resolve closure during codegen");
@@ -875,12 +875,11 @@ fn visit_fn_use<'tcx>(
source: Span,
output: &mut MonoItems<'tcx>,
) {
- if let ty::FnDef(def_id, substs) = *ty.kind() {
+ if let ty::FnDef(def_id, args) = *ty.kind() {
let instance = if is_direct_call {
- ty::Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, substs)
+ ty::Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args)
} else {
- match ty::Instance::resolve_for_fn_ptr(tcx, ty::ParamEnv::reveal_all(), def_id, substs)
- {
+ match ty::Instance::resolve_for_fn_ptr(tcx, ty::ParamEnv::reveal_all(), def_id, args) {
Some(instance) => instance,
_ => bug!("failed to resolve instance for {ty}"),
}
@@ -1043,7 +1042,7 @@ fn find_vtable_types_for_unsizing<'tcx>(
// T as dyn* Trait
(_, &ty::Dynamic(_, _, ty::DynStar)) => ptr_vtable(source_ty, target_ty),
- (&ty::Adt(source_adt_def, source_substs), &ty::Adt(target_adt_def, target_substs)) => {
+ (&ty::Adt(source_adt_def, source_args), &ty::Adt(target_adt_def, target_args)) => {
assert_eq!(source_adt_def, target_adt_def);
let CustomCoerceUnsized::Struct(coerce_index) =
@@ -1059,8 +1058,8 @@ fn find_vtable_types_for_unsizing<'tcx>(
find_vtable_types_for_unsizing(
tcx,
- source_fields[coerce_index].ty(*tcx, source_substs),
- target_fields[coerce_index].ty(*tcx, target_substs),
+ source_fields[coerce_index].ty(*tcx, source_args),
+ target_fields[coerce_index].ty(*tcx, target_args),
)
}
_ => bug!(
@@ -1245,7 +1244,7 @@ impl<'v> RootCollector<'_, 'v> {
self.tcx,
ty::ParamEnv::reveal_all(),
start_def_id,
- self.tcx.mk_substs(&[main_ret_ty.into()]),
+ self.tcx.mk_args(&[main_ret_ty.into()]),
)
.unwrap()
.unwrap();
@@ -1292,8 +1291,8 @@ fn create_mono_items_for_default_impls<'tcx>(
)
}
};
- let impl_substs = InternalSubsts::for_item(tcx, item.owner_id.to_def_id(), only_region_params);
- let trait_ref = trait_ref.subst(tcx, impl_substs);
+ let impl_args = GenericArgs::for_item(tcx, item.owner_id.to_def_id(), only_region_params);
+ let trait_ref = trait_ref.instantiate(tcx, impl_args);
// Unlike 'lazy' monomorphization that begins by collecting items transitively
// called by `main` or other global items, when eagerly monomorphizing impl
@@ -1304,7 +1303,7 @@ fn create_mono_items_for_default_impls<'tcx>(
// consider higher-ranked predicates such as `for<'a> &'a mut [u8]: Copy` to
// be trivially false. We must now check that the impl has no impossible-to-satisfy
// predicates.
- if tcx.subst_and_check_impossible_predicates((item.owner_id.to_def_id(), impl_substs)) {
+ if tcx.subst_and_check_impossible_predicates((item.owner_id.to_def_id(), impl_args)) {
return;
}
@@ -1322,8 +1321,8 @@ fn create_mono_items_for_default_impls<'tcx>(
// As mentioned above, the method is legal to eagerly instantiate if it
// only has lifetime substitutions. This is validated by
- let substs = trait_ref.substs.extend_to(tcx, method.def_id, only_region_params);
- let instance = ty::Instance::expect_resolve(tcx, param_env, method.def_id, substs);
+ let args = trait_ref.args.extend_to(tcx, method.def_id, only_region_params);
+ let instance = ty::Instance::expect_resolve(tcx, param_env, method.def_id, args);
let mono_item = create_fn_mono_item(tcx, instance, DUMMY_SP);
if mono_item.node.is_instantiable(tcx) && should_codegen_locally(tcx, &instance) {
@@ -1332,8 +1331,8 @@ fn create_mono_items_for_default_impls<'tcx>(
}
}
-/// Scans the miri alloc in order to find function calls, closures, and drop-glue.
-fn collect_miri<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut MonoItems<'tcx>) {
+/// Scans the CTFE alloc in order to find function calls, closures, and drop-glue.
+fn collect_alloc<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut MonoItems<'tcx>) {
match tcx.global_alloc(alloc_id) {
GlobalAlloc::Static(def_id) => {
assert!(!tcx.is_thread_local_static(def_id));
@@ -1347,7 +1346,7 @@ fn collect_miri<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut MonoIte
trace!("collecting {:?} with {:#?}", alloc_id, alloc);
for &inner in alloc.inner().provenance().ptrs().values() {
rustc_data_structures::stack::ensure_sufficient_stack(|| {
- collect_miri(tcx, inner, output);
+ collect_alloc(tcx, inner, output);
});
}
}
@@ -1359,7 +1358,7 @@ fn collect_miri<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut MonoIte
}
GlobalAlloc::VTable(ty, trait_ref) => {
let alloc_id = tcx.vtable_allocation((ty, trait_ref));
- collect_miri(tcx, alloc_id, output)
+ collect_alloc(tcx, alloc_id, output)
}
}
}
@@ -1382,10 +1381,10 @@ fn collect_const_value<'tcx>(
output: &mut MonoItems<'tcx>,
) {
match value {
- ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => collect_miri(tcx, ptr.provenance, output),
+ ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => collect_alloc(tcx, ptr.provenance, output),
ConstValue::Slice { data: alloc, start: _, end: _ } | ConstValue::ByRef { alloc, .. } => {
for &id in alloc.inner().provenance().ptrs().values() {
- collect_miri(tcx, id, output);
+ collect_alloc(tcx, id, output);
}
}
_ => {}
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index da76cf223..de6db8ae6 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -107,7 +107,8 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
use rustc_middle::mir;
use rustc_middle::mir::mono::{
- CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, Visibility,
+ CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, MonoItemData,
+ Visibility,
};
use rustc_middle::query::Providers;
use rustc_middle::ty::print::{characteristic_def_id_of_type, with_no_trimmed_paths};
@@ -130,11 +131,6 @@ struct PlacedMonoItems<'tcx> {
codegen_units: Vec<CodegenUnit<'tcx>>,
internalization_candidates: FxHashSet<MonoItem<'tcx>>,
-
- /// These must be obtained when the iterator in `partition` runs. They
- /// can't be obtained later because some inlined functions might not be
- /// reachable.
- unique_inlined_stats: (usize, usize),
}
// The output CGUs are sorted by name.
@@ -152,11 +148,11 @@ where
// Place all mono items into a codegen unit. `place_mono_items` is
// responsible for initializing the CGU size estimates.
- let PlacedMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
+ let PlacedMonoItems { mut codegen_units, internalization_candidates } = {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_items");
let placed = place_mono_items(cx, mono_items);
- debug_dump(tcx, "PLACE", &placed.codegen_units, placed.unique_inlined_stats);
+ debug_dump(tcx, "PLACE", &placed.codegen_units);
placed
};
@@ -167,7 +163,7 @@ where
{
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_merge_cgus");
merge_codegen_units(cx, &mut codegen_units);
- debug_dump(tcx, "MERGE", &codegen_units, unique_inlined_stats);
+ debug_dump(tcx, "MERGE", &codegen_units);
}
// Make as many symbols "internal" as possible, so LLVM has more freedom to
@@ -176,7 +172,7 @@ where
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols");
internalize_symbols(cx, &mut codegen_units, internalization_candidates);
- debug_dump(tcx, "INTERNALIZE", &codegen_units, unique_inlined_stats);
+ debug_dump(tcx, "INTERNALIZE", &codegen_units);
}
// Mark one CGU for dead code, if necessary.
@@ -216,18 +212,12 @@ where
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
let cgu_name_cache = &mut FxHashMap::default();
- let mut num_unique_inlined_items = 0;
- let mut unique_inlined_items_size = 0;
for mono_item in mono_items {
// Handle only root items directly here. Inlined items are handled at
// the bottom of the loop based on reachability.
match mono_item.instantiation_mode(cx.tcx) {
InstantiationMode::GloballyShared { .. } => {}
- InstantiationMode::LocalCopy => {
- num_unique_inlined_items += 1;
- unique_inlined_items_size += mono_item.size_estimate(cx.tcx);
- continue;
- }
+ InstantiationMode::LocalCopy => continue,
}
let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
@@ -256,8 +246,10 @@ where
if visibility == Visibility::Hidden && can_be_internalized {
internalization_candidates.insert(mono_item);
}
+ let size_estimate = mono_item.size_estimate(cx.tcx);
- cgu.items_mut().insert(mono_item, (linkage, visibility));
+ cgu.items_mut()
+ .insert(mono_item, MonoItemData { inlined: false, linkage, visibility, size_estimate });
// Get all inlined items that are reachable from `mono_item` without
// going via another root item. This includes drop-glue, functions from
@@ -271,7 +263,12 @@ where
// the `insert` will be a no-op.
for inlined_item in reachable_inlined_items {
// This is a CGU-private copy.
- cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default));
+ cgu.items_mut().entry(inlined_item).or_insert_with(|| MonoItemData {
+ inlined: true,
+ linkage: Linkage::Internal,
+ visibility: Visibility::Default,
+ size_estimate: inlined_item.size_estimate(cx.tcx),
+ });
}
}
@@ -286,14 +283,10 @@ where
codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
for cgu in codegen_units.iter_mut() {
- cgu.compute_size_estimate(cx.tcx);
+ cgu.compute_size_estimate();
}
- return PlacedMonoItems {
- codegen_units,
- internalization_candidates,
- unique_inlined_stats: (num_unique_inlined_items, unique_inlined_items_size),
- };
+ return PlacedMonoItems { codegen_units, internalization_candidates };
fn get_reachable_inlined_items<'tcx>(
tcx: TyCtxt<'tcx>,
@@ -325,6 +318,60 @@ fn merge_codegen_units<'tcx>(
let mut cgu_contents: FxHashMap<Symbol, Vec<Symbol>> =
codegen_units.iter().map(|cgu| (cgu.name(), vec![cgu.name()])).collect();
+ // If N is the maximum number of CGUs, and the CGUs are sorted from largest
+ // to smallest, we repeatedly find which CGU in codegen_units[N..] has the
+ // greatest overlap of inlined items with codegen_units[N-1], merge that
+ // CGU into codegen_units[N-1], then re-sort by size and repeat.
+ //
+ // We use inlined item overlap to guide this merging because it minimizes
+ // duplication of inlined items, which makes LLVM be faster and generate
+ // better and smaller machine code.
+ //
+ // Why merge into codegen_units[N-1]? We want CGUs to have similar sizes,
+ // which means we don't want codegen_units[0..N] (the already big ones)
+ // getting any bigger, if we can avoid it. When we have more than N CGUs
+ // then at least one of the biggest N will have to grow. codegen_units[N-1]
+ // is the smallest of those, and so has the most room to grow.
+ let max_codegen_units = cx.tcx.sess.codegen_units().as_usize();
+ while codegen_units.len() > max_codegen_units {
+ // Sort small CGUs to the back.
+ codegen_units.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));
+
+ let cgu_dst = &codegen_units[max_codegen_units - 1];
+
+ // Find the CGU that overlaps the most with `cgu_dst`. In the case of a
+ // tie, favour the earlier (bigger) CGU.
+ let mut max_overlap = 0;
+ let mut max_overlap_i = max_codegen_units;
+ for (i, cgu_src) in codegen_units.iter().enumerate().skip(max_codegen_units) {
+ if cgu_src.size_estimate() <= max_overlap {
+ // None of the remaining overlaps can exceed `max_overlap`, so
+ // stop looking.
+ break;
+ }
+
+ let overlap = compute_inlined_overlap(cgu_dst, cgu_src);
+ if overlap > max_overlap {
+ max_overlap = overlap;
+ max_overlap_i = i;
+ }
+ }
+
+ let mut cgu_src = codegen_units.swap_remove(max_overlap_i);
+ let cgu_dst = &mut codegen_units[max_codegen_units - 1];
+
+ // Move the items from `cgu_src` to `cgu_dst`. Some of them may be
+ // duplicate inlined items, in which case the destination CGU is
+ // unaffected. Recalculate size estimates afterwards.
+ cgu_dst.items_mut().extend(cgu_src.items_mut().drain());
+ cgu_dst.compute_size_estimate();
+
+ // Record that `cgu_dst` now contains all the stuff that was in
+ // `cgu_src` before.
+ let mut consumed_cgu_names = cgu_contents.remove(&cgu_src.name()).unwrap();
+ cgu_contents.get_mut(&cgu_dst.name()).unwrap().append(&mut consumed_cgu_names);
+ }
+
// Having multiple CGUs can drastically speed up compilation. But for
// non-incremental builds, tiny CGUs slow down compilation *and* result in
// worse generated code. So we don't allow CGUs smaller than this (unless
@@ -332,24 +379,22 @@ fn merge_codegen_units<'tcx>(
// common in larger programs, so this isn't all that large.
const NON_INCR_MIN_CGU_SIZE: usize = 1800;
- // Repeatedly merge the two smallest codegen units as long as:
- // - we have more CGUs than the upper limit, or
- // - (Non-incremental builds only) the user didn't specify a CGU count, and
- // there are multiple CGUs, and some are below the minimum size.
+ // Repeatedly merge the two smallest codegen units as long as: it's a
+ // non-incremental build, and the user didn't specify a CGU count, and
+ // there are multiple CGUs, and some are below the minimum size.
//
// The "didn't specify a CGU count" condition is because when an explicit
// count is requested we observe it as closely as possible. For example,
// the `compiler_builtins` crate sets `codegen-units = 10000` and it's
// critical they aren't merged. Also, some tests use explicit small values
// and likewise won't work if small CGUs are merged.
- while codegen_units.len() > cx.tcx.sess.codegen_units().as_usize()
- || (cx.tcx.sess.opts.incremental.is_none()
- && matches!(cx.tcx.sess.codegen_units(), CodegenUnits::Default(_))
- && codegen_units.len() > 1
- && codegen_units.iter().any(|cgu| cgu.size_estimate() < NON_INCR_MIN_CGU_SIZE))
+ while cx.tcx.sess.opts.incremental.is_none()
+ && matches!(cx.tcx.sess.codegen_units(), CodegenUnits::Default(_))
+ && codegen_units.len() > 1
+ && codegen_units.iter().any(|cgu| cgu.size_estimate() < NON_INCR_MIN_CGU_SIZE)
{
// Sort small cgus to the back.
- codegen_units.sort_by_cached_key(|cgu| cmp::Reverse(cgu.size_estimate()));
+ codegen_units.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));
let mut smallest = codegen_units.pop().unwrap();
let second_smallest = codegen_units.last_mut().unwrap();
@@ -358,18 +403,9 @@ fn merge_codegen_units<'tcx>(
// may be duplicate inlined items, in which case the destination CGU is
// unaffected. Recalculate size estimates afterwards.
second_smallest.items_mut().extend(smallest.items_mut().drain());
- second_smallest.compute_size_estimate(cx.tcx);
-
- // Record that `second_smallest` now contains all the stuff that was
- // in `smallest` before.
- let mut consumed_cgu_names = cgu_contents.remove(&smallest.name()).unwrap();
- cgu_contents.get_mut(&second_smallest.name()).unwrap().append(&mut consumed_cgu_names);
+ second_smallest.compute_size_estimate();
- debug!(
- "CodegenUnit {} merged into CodegenUnit {}",
- smallest.name(),
- second_smallest.name()
- );
+ // Don't update `cgu_contents`, that's only for incremental builds.
}
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
@@ -448,6 +484,25 @@ fn merge_codegen_units<'tcx>(
}
}
+/// Compute the combined size of all inlined items that appear in both `cgu1`
+/// and `cgu2`.
+fn compute_inlined_overlap<'tcx>(cgu1: &CodegenUnit<'tcx>, cgu2: &CodegenUnit<'tcx>) -> usize {
+ // Either order works. We pick the one that involves iterating over fewer
+ // items.
+ let (src_cgu, dst_cgu) =
+ if cgu1.items().len() <= cgu2.items().len() { (cgu1, cgu2) } else { (cgu2, cgu1) };
+
+ let mut overlap = 0;
+ for (item, data) in src_cgu.items().iter() {
+ if data.inlined {
+ if dst_cgu.items().contains_key(item) {
+ overlap += data.size_estimate;
+ }
+ }
+ }
+ overlap
+}
+
fn internalize_symbols<'tcx>(
cx: &PartitioningCx<'_, 'tcx>,
codegen_units: &mut [CodegenUnit<'tcx>],
@@ -492,7 +547,7 @@ fn internalize_symbols<'tcx>(
for cgu in codegen_units {
let home_cgu = MonoItemPlacement::SingleCgu(cgu.name());
- for (item, linkage_and_visibility) in cgu.items_mut() {
+ for (item, data) in cgu.items_mut() {
if !internalization_candidates.contains(item) {
// This item is no candidate for internalizing, so skip it.
continue;
@@ -520,7 +575,8 @@ fn internalize_symbols<'tcx>(
// If we got here, we did not find any uses from other CGUs, so
// it's fine to make this monomorphization internal.
- *linkage_and_visibility = (Linkage::Internal, Visibility::Default);
+ data.linkage = Linkage::Internal;
+ data.visibility = Visibility::Default;
}
}
}
@@ -537,7 +593,7 @@ fn mark_code_coverage_dead_code_cgu<'tcx>(codegen_units: &mut [CodegenUnit<'tcx>
// function symbols to be included via `-u` or `/include` linker args.
let dead_code_cgu = codegen_units
.iter_mut()
- .filter(|cgu| cgu.items().iter().any(|(_, (linkage, _))| *linkage == Linkage::External))
+ .filter(|cgu| cgu.items().iter().any(|(_, data)| data.linkage == Linkage::External))
.min_by_key(|cgu| cgu.size_estimate());
// If there are no CGUs that have externally linked items, then we just
@@ -572,7 +628,7 @@ fn characteristic_def_id_of_mono_item<'tcx>(
// DefId, we use the location of the impl after all.
if tcx.trait_of_item(def_id).is_some() {
- let self_ty = instance.substs.type_at(0);
+ let self_ty = instance.args.type_at(0);
// This is a default implementation of a trait method.
return characteristic_def_id_of_type(self_ty).or(Some(def_id));
}
@@ -592,7 +648,7 @@ fn characteristic_def_id_of_mono_item<'tcx>(
if !tcx.sess.opts.unstable_opts.polymorphize || !instance.has_param() {
// This is a method within an impl, find out what the self-type is:
let impl_self_ty = tcx.subst_and_normalize_erasing_regions(
- instance.substs,
+ instance.args,
ty::ParamEnv::reveal_all(),
tcx.type_of(impl_def_id),
);
@@ -745,7 +801,7 @@ fn mono_item_visibility<'tcx>(
return Visibility::Hidden;
}
- let is_generic = instance.substs.non_erasable_generics().next().is_some();
+ let is_generic = instance.args.non_erasable_generics().next().is_some();
// Upstream `DefId` instances get different handling than local ones.
let Some(def_id) = def_id.as_local() else {
@@ -851,12 +907,7 @@ fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibilit
}
}
-fn debug_dump<'a, 'tcx: 'a>(
- tcx: TyCtxt<'tcx>,
- label: &str,
- cgus: &[CodegenUnit<'tcx>],
- (unique_inlined_items, unique_inlined_size): (usize, usize),
-) {
+fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit<'tcx>]) {
let dump = move || {
use std::fmt::Write;
@@ -865,29 +916,34 @@ fn debug_dump<'a, 'tcx: 'a>(
// Note: every unique root item is placed exactly once, so the number
// of unique root items always equals the number of placed root items.
+ //
+ // Also, unreached inlined items won't be counted here. This is fine.
+
+ let mut inlined_items = FxHashSet::default();
let mut root_items = 0;
- // unique_inlined_items is passed in above.
+ let mut unique_inlined_items = 0;
let mut placed_inlined_items = 0;
let mut root_size = 0;
- // unique_inlined_size is passed in above.
+ let mut unique_inlined_size = 0;
let mut placed_inlined_size = 0;
for cgu in cgus.iter() {
num_cgus += 1;
all_cgu_sizes.push(cgu.size_estimate());
- for (item, _) in cgu.items() {
- match item.instantiation_mode(tcx) {
- InstantiationMode::GloballyShared { .. } => {
- root_items += 1;
- root_size += item.size_estimate(tcx);
- }
- InstantiationMode::LocalCopy => {
- placed_inlined_items += 1;
- placed_inlined_size += item.size_estimate(tcx);
+ for (item, data) in cgu.items() {
+ if !data.inlined {
+ root_items += 1;
+ root_size += data.size_estimate;
+ } else {
+ if inlined_items.insert(item) {
+ unique_inlined_items += 1;
+ unique_inlined_size += data.size_estimate;
}
+ placed_inlined_items += 1;
+ placed_inlined_size += data.size_estimate;
}
}
}
@@ -928,7 +984,7 @@ fn debug_dump<'a, 'tcx: 'a>(
let mean_size = size as f64 / num_items as f64;
let mut placed_item_sizes: Vec<_> =
- cgu.items().iter().map(|(item, _)| item.size_estimate(tcx)).collect();
+ cgu.items().values().map(|data| data.size_estimate).collect();
placed_item_sizes.sort_unstable_by_key(|&n| cmp::Reverse(n));
let sizes = list(&placed_item_sizes);
@@ -937,15 +993,13 @@ fn debug_dump<'a, 'tcx: 'a>(
let _ =
writeln!(s, " - items: {num_items}, mean size: {mean_size:.1}, sizes: {sizes}",);
- for (item, linkage) in cgu.items_in_deterministic_order(tcx) {
+ for (item, data) in cgu.items_in_deterministic_order(tcx) {
+ let linkage = data.linkage;
let symbol_name = item.symbol_name(tcx).name;
let symbol_hash_start = symbol_name.rfind('h');
let symbol_hash = symbol_hash_start.map_or("<no hash>", |i| &symbol_name[i..]);
- let size = item.size_estimate(tcx);
- let kind = match item.instantiation_mode(tcx) {
- InstantiationMode::GloballyShared { .. } => "root",
- InstantiationMode::LocalCopy => "inlined",
- };
+ let kind = if !data.inlined { "root" } else { "inlined" };
+ let size = data.size_estimate;
let _ = with_no_trimmed_paths!(writeln!(
s,
" - {item} [{linkage:?}] [{symbol_hash}] ({kind}, size: {size})"
@@ -987,10 +1041,7 @@ fn debug_dump<'a, 'tcx: 'a>(
}
elem(curr, curr_count);
- let mut s = "[".to_string();
- s.push_str(&v.join(", "));
- s.push_str("]");
- s
+ format!("[{}]", v.join(", "))
}
};
@@ -1100,8 +1151,8 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
let mut item_to_cgus: FxHashMap<_, Vec<_>> = Default::default();
for cgu in codegen_units {
- for (&mono_item, &linkage) in cgu.items() {
- item_to_cgus.entry(mono_item).or_default().push((cgu.name(), linkage));
+ for (&mono_item, &data) in cgu.items() {
+ item_to_cgus.entry(mono_item).or_default().push((cgu.name(), data.linkage));
}
}
@@ -1114,7 +1165,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
let cgus = item_to_cgus.get_mut(i).unwrap_or(&mut empty);
cgus.sort_by_key(|(name, _)| *name);
cgus.dedup();
- for &(ref cgu_name, (linkage, _)) in cgus.iter() {
+ for &(ref cgu_name, linkage) in cgus.iter() {
output.push(' ');
output.push_str(cgu_name.as_str());
@@ -1175,12 +1226,13 @@ fn dump_mono_items_stats<'tcx>(
// Gather instantiated mono items grouped by def_id
let mut items_per_def_id: FxHashMap<_, Vec<_>> = Default::default();
for cgu in codegen_units {
- for (&mono_item, _) in cgu.items() {
+ cgu.items()
+ .keys()
// Avoid variable-sized compiler-generated shims
- if mono_item.is_user_defined() {
+ .filter(|mono_item| mono_item.is_user_defined())
+ .for_each(|mono_item| {
items_per_def_id.entry(mono_item.def_id()).or_default().push(mono_item);
- }
- }
+ });
}
#[derive(serde::Serialize)]
@@ -1233,7 +1285,7 @@ fn codegened_and_inlined_items(tcx: TyCtxt<'_>, (): ()) -> &DefIdSet {
let mut result = items.clone();
for cgu in cgus {
- for (item, _) in cgu.items() {
+ for item in cgu.items().keys() {
if let MonoItem::Fn(ref instance) = item {
let did = instance.def_id();
if !visited.insert(did) {
diff --git a/compiler/rustc_monomorphize/src/polymorphize.rs b/compiler/rustc_monomorphize/src/polymorphize.rs
index 88a3e0285..a8b7a0dbb 100644
--- a/compiler/rustc_monomorphize/src/polymorphize.rs
+++ b/compiler/rustc_monomorphize/src/polymorphize.rs
@@ -14,9 +14,8 @@ use rustc_middle::mir::{
use rustc_middle::query::Providers;
use rustc_middle::ty::{
self,
- subst::SubstsRef,
visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor},
- Const, Ty, TyCtxt, UnusedGenericParams,
+ Const, GenericArgsRef, Ty, TyCtxt, UnusedGenericParams,
};
use rustc_span::symbol::sym;
use std::ops::ControlFlow;
@@ -144,7 +143,7 @@ fn mark_used_by_default_parameters<'tcx>(
| DefKind::Enum
| DefKind::Variant
| DefKind::Trait
- | DefKind::TyAlias
+ | DefKind::TyAlias { .. }
| DefKind::ForeignTy
| DefKind::TraitAlias
| DefKind::AssocTy
@@ -163,7 +162,6 @@ fn mark_used_by_default_parameters<'tcx>(
| DefKind::AnonConst
| DefKind::InlineConst
| DefKind::OpaqueTy
- | DefKind::ImplTraitPlaceholder
| DefKind::Field
| DefKind::LifetimeParam
| DefKind::GlobalAsm
@@ -230,12 +228,12 @@ struct MarkUsedGenericParams<'a, 'tcx> {
impl<'a, 'tcx> MarkUsedGenericParams<'a, 'tcx> {
/// Invoke `unused_generic_params` on a body contained within the current item (e.g.
/// a closure, generator or constant).
- #[instrument(level = "debug", skip(self, def_id, substs))]
- fn visit_child_body(&mut self, def_id: DefId, substs: SubstsRef<'tcx>) {
+ #[instrument(level = "debug", skip(self, def_id, args))]
+ fn visit_child_body(&mut self, def_id: DefId, args: GenericArgsRef<'tcx>) {
let instance = ty::InstanceDef::Item(def_id);
let unused = self.tcx.unused_generic_params(instance);
debug!(?self.unused_parameters, ?unused);
- for (i, arg) in substs.iter().enumerate() {
+ for (i, arg) in args.iter().enumerate() {
let i = i.try_into().unwrap();
if unused.is_used(i) {
arg.visit_with(self);
@@ -253,9 +251,9 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
if matches!(def_kind, DefKind::Closure | DefKind::Generator) {
// Skip visiting the closure/generator that is currently being processed. This only
// happens because the first argument to the closure is a reference to itself and
- // that will call `visit_substs`, resulting in each generic parameter captured being
+ // that will call `visit_args`, resulting in each generic parameter captured being
// considered used by default.
- debug!("skipping closure substs");
+ debug!("skipping closure args");
return;
}
}
@@ -268,12 +266,12 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
ConstantKind::Ty(c) => {
c.visit_with(self);
}
- ConstantKind::Unevaluated(mir::UnevaluatedConst { def, substs: _, promoted }, ty) => {
+ ConstantKind::Unevaluated(mir::UnevaluatedConst { def, args: _, promoted }, ty) => {
// Avoid considering `T` unused when constants are of the form:
// `<Self as Foo<T>>::foo::promoted[p]`
if let Some(p) = promoted {
if self.def_id == def && !self.tcx.generics_of(def).has_self {
- // If there is a promoted, don't look at the substs - since it will always contain
+ // If there is a promoted, don't look at the args - since it will always contain
// the generic parameters, instead, traverse the promoted MIR.
let promoted = self.tcx.promoted_mir(def);
self.visit_body(&promoted[p]);
@@ -304,10 +302,10 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for MarkUsedGenericParams<'a, 'tcx> {
self.unused_parameters.mark_used(param.index);
ControlFlow::Continue(())
}
- ty::ConstKind::Unevaluated(ty::UnevaluatedConst { def, substs })
+ ty::ConstKind::Unevaluated(ty::UnevaluatedConst { def, args })
if matches!(self.tcx.def_kind(def), DefKind::AnonConst) =>
{
- self.visit_child_body(def, substs);
+ self.visit_child_body(def, args);
ControlFlow::Continue(())
}
_ => c.super_visit_with(self),
@@ -321,7 +319,7 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for MarkUsedGenericParams<'a, 'tcx> {
}
match *ty.kind() {
- ty::Closure(def_id, substs) | ty::Generator(def_id, substs, ..) => {
+ ty::Closure(def_id, args) | ty::Generator(def_id, args, ..) => {
debug!(?def_id);
// Avoid cycle errors with generators.
if def_id == self.def_id {
@@ -330,7 +328,7 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for MarkUsedGenericParams<'a, 'tcx> {
// Consider any generic parameters used by any closures/generators as used in the
// parent.
- self.visit_child_body(def_id, substs);
+ self.visit_child_body(def_id, args);
ControlFlow::Continue(())
}
ty::Param(param) => {
diff --git a/compiler/rustc_monomorphize/src/util.rs b/compiler/rustc_monomorphize/src/util.rs
index f6a80b043..a3433d3d1 100644
--- a/compiler/rustc_monomorphize/src/util.rs
+++ b/compiler/rustc_monomorphize/src/util.rs
@@ -27,12 +27,12 @@ pub(crate) fn dump_closure_profile<'tcx>(tcx: TyCtxt<'tcx>, closure_instance: In
typeck_results.closure_size_eval[&closure_def_id];
let before_feature_tys = tcx.subst_and_normalize_erasing_regions(
- closure_instance.substs,
+ closure_instance.args,
param_env,
ty::EarlyBinder::bind(before_feature_tys),
);
let after_feature_tys = tcx.subst_and_normalize_erasing_regions(
- closure_instance.substs,
+ closure_instance.args,
param_env,
ty::EarlyBinder::bind(after_feature_tys),
);