Auto merge of rust-lang#131650 - saethlin:post-mono-mir-opts, r=<try>

Add post-mono MIR passes to make mono-reachable analysis more accurate

As of rust-lang#131650 (comment) I believe most of the incr overhead comes from re-computing, re-encoding, and loading a lot more MIR when all we're actually doing is traversing through it. I think that can be addressed by caching a query that looks up the mentioned/used items for an Instance.

I think the full-build regressions are pretty much just the expense of cloning, then monomorphizing, then caching the MIR.

compiler/rustc_codegen_cranelift/src/base.rs

@@ -41,7 +41,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let symbol_name = tcx.symbol_name(instance).name.to_string();
     let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
 
-    let mir = tcx.instance_mir(instance.def);
+    let mir = tcx.codegen_mir(instance);
     let _mir_guard = crate::PrintOnPanic(|| {
         let mut buf = Vec::new();
         with_no_trimmed_paths!({

compiler/rustc_codegen_ssa/src/base.rs

@@ -416,7 +416,7 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
     // release builds.
     info!("codegen_instance({})", instance);
 
-    mir::codegen_mir::<Bx>(cx, instance);
+    mir::lower_mir::<Bx>(cx, instance);
 }
 
 /// Creates the `main` function which will initialize the rust runtime and call

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -159,15 +159,15 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
 ///////////////////////////////////////////////////////////////////////////
 
 #[instrument(level = "debug", skip(cx))]
-pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+pub fn lower_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     cx: &'a Bx::CodegenCx,
     instance: Instance<'tcx>,
 ) {
     assert!(!instance.args.has_infer());
 
     let llfn = cx.get_fn(instance);
 
-    let mir = cx.tcx().instance_mir(instance.def);
+    let mir = cx.tcx().codegen_mir(instance);
 
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
     debug!("fn_abi: {:?}", fn_abi);

compiler/rustc_middle/src/mir/mono.rs

@@ -73,7 +73,7 @@ impl<'tcx> MonoItem<'tcx> {
                     InstanceKind::Item(..)
                     | InstanceKind::DropGlue(..)
                     | InstanceKind::AsyncDropGlueCtorShim(..) => {
-                        let mir = tcx.instance_mir(instance.def);
+                        let mir = tcx.codegen_mir(instance);
                         mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum()
                     }
                     // Other compiler-generated shims size estimate: 1

compiler/rustc_middle/src/query/mod.rs

@@ -555,14 +555,21 @@ rustc_queries! {
         desc { |tcx| "verify auto trait bounds for coroutine interior type `{}`", tcx.def_path_str(key) }
     }
 
-    /// MIR after our optimization passes have run. This is MIR that is ready
-    /// for codegen. This is also the only query that can fetch non-local MIR, at present.
+    /// Polymorphic MIR after our pre-mono optimization passes have run. This is the MIR that
+    /// crates export.
     query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
         desc { |tcx| "optimizing MIR for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
 
+    /// MIR for a specific Instance ready for codegen. This is `optimized_mir` but monomorphized
+    /// and with extra transforms applied.
+    query build_codegen_mir(key: ty::Instance<'tcx>) -> &'tcx mir::Body<'tcx> {
+        desc { |tcx| "finalizing codegen MIR for `{}`", tcx.def_path_str_with_args(key.def_id(), key.args) }
+        arena_cache
+    }
+
     /// Checks for the nearest `#[coverage(off)]` or `#[coverage(on)]` on
     /// this def and any enclosing defs, up to the crate root.
     ///

compiler/rustc_middle/src/ty/mod.rs

@@ -1719,6 +1719,14 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    pub fn codegen_mir(self, instance: ty::Instance<'tcx>) -> &'tcx Body<'tcx> {
+        if self.sess.opts.incremental == None {
+            self.build_codegen_mir(instance)
+        } else {
+            self.instance_mir(instance.def)
+        }
+    }
+
     /// Returns the possibly-auto-generated MIR of a [`ty::InstanceKind`].
     #[instrument(skip(self), level = "debug")]
     pub fn instance_mir(self, instance: ty::InstanceKind<'tcx>) -> &'tcx Body<'tcx> {

compiler/rustc_mir_transform/src/abort_unwinding_calls.rs

@@ -1,7 +1,7 @@
 use rustc_ast::InlineAsmOptions;
 use rustc_middle::mir::*;
 use rustc_middle::span_bug;
-use rustc_middle::ty::{self, TyCtxt, layout};
+use rustc_middle::ty::{self, ParamEnv, TyCtxt, layout};
 use rustc_target::spec::PanicStrategy;
 use rustc_target::spec::abi::Abi;
 
@@ -23,7 +23,7 @@ use rustc_target::spec::abi::Abi;
 pub(super) struct AbortUnwindingCalls;
 
 impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         let def_id = body.source.def_id();
         let kind = tcx.def_kind(def_id);
 

compiler/rustc_mir_transform/src/add_call_guards.rs

@@ -1,6 +1,6 @@
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::debug;
 
 #[derive(PartialEq)]
@@ -31,7 +31,7 @@ pub(super) use self::AddCallGuards::*;
  */
 
 impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
-    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         let mut pred_count: IndexVec<_, _> =
             body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect();
         pred_count[START_BLOCK] += 1;

compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs

@@ -1,6 +1,6 @@
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::debug;
 
 use crate::util;
@@ -38,12 +38,10 @@ use crate::util;
 pub(super) struct AddMovesForPackedDrops;
 
 impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         debug!("add_moves_for_packed_drops({:?} @ {:?})", body.source, body.span);
 
-        let def_id = body.source.def_id();
         let mut patch = MirPatch::new(body);
-        let param_env = tcx.param_env(def_id);
 
         for (bb, data) in body.basic_blocks.iter_enumerated() {
             let loc = Location { block: bb, statement_index: data.statements.len() };

compiler/rustc_mir_transform/src/add_retag.rs

@@ -6,7 +6,7 @@
 
 use rustc_hir::LangItem;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
 
 pub(super) struct AddRetag;
 
@@ -53,9 +53,9 @@ impl<'tcx> crate::MirPass<'tcx> for AddRetag {
         sess.opts.unstable_opts.mir_emit_retag
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         // We need an `AllCallEdges` pass before we can do any work.
-        super::add_call_guards::AllCallEdges.run_pass(tcx, body);
+        super::add_call_guards::AllCallEdges.run_pass(tcx, param_env, body);
 
         let basic_blocks = body.basic_blocks.as_mut();
         let local_decls = &body.local_decls;

compiler/rustc_mir_transform/src/add_subtyping_projections.rs

@@ -1,7 +1,7 @@
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 
 pub(super) struct Subtyper;
 
@@ -52,7 +52,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for SubTypeChecker<'a, 'tcx> {
 // let temp: rval_ty = rval;
 // let place: place_ty = temp as place_ty;
 impl<'tcx> crate::MirPass<'tcx> for Subtyper {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         let patch = MirPatch::new(body);
         let mut checker = SubTypeChecker { tcx, patcher: patch, local_decls: &body.local_decls };
 

compiler/rustc_mir_transform/src/check_alignment.rs

@@ -18,7 +18,7 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment {
         sess.ub_checks()
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         // This pass emits new panics. If for whatever reason we do not have a panic
         // implementation, running this pass may cause otherwise-valid code to not compile.
         if tcx.lang_items().get(LangItem::PanicImpl).is_none() {
@@ -27,7 +27,6 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment {
 
         let basic_blocks = body.basic_blocks.as_mut();
         let local_decls = &mut body.local_decls;
-        let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
 
         // This pass inserts new blocks. Each insertion changes the Location for all
         // statements/blocks after. Iterating or visiting the MIR in order would require updating

compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs

@@ -18,13 +18,13 @@
 
 use rustc_middle::mir::coverage::CoverageKind;
 use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, TerminatorKind};
-use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::adjustment::PointerCoercion;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 
 pub(super) struct CleanupPostBorrowck;
 
 impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck {
-    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         for basic_block in body.basic_blocks.as_mut() {
             for statement in basic_block.statements.iter_mut() {
                 match statement.kind {

compiler/rustc_mir_transform/src/copy_prop.rs

@@ -2,7 +2,7 @@ use rustc_index::IndexSlice;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::{debug, instrument};
 
 use crate::ssa::SsaLocals;
@@ -25,10 +25,9 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp {
     }
 
     #[instrument(level = "trace", skip(self, tcx, body))]
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         debug!(def_id = ?body.source.def_id());
 
-        let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         let ssa = SsaLocals::new(tcx, body, param_env);
 
         let fully_moved = fully_moved_locals(&ssa, body);

compiler/rustc_mir_transform/src/coroutine.rs

@@ -64,7 +64,7 @@ use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::{
-    self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt,
+    self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, ParamEnv, Ty, TyCtxt,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_mir_dataflow::Analysis;
@@ -1361,7 +1361,13 @@ fn create_coroutine_resume_function<'tcx>(
     // unrelated code from the drop part of the function
     simplify::remove_dead_blocks(body);
 
-    pm::run_passes_no_validate(tcx, body, &[&abort_unwinding_calls::AbortUnwindingCalls], None);
+    pm::run_passes_no_validate(
+        tcx,
+        tcx.param_env_reveal_all_normalized(body.source.def_id()),
+        body,
+        &[&abort_unwinding_calls::AbortUnwindingCalls],
+        None,
+    );
 
     dump_mir(tcx, false, "coroutine_resume", &0, body, |_, _| Ok(()));
 }
@@ -1526,7 +1532,7 @@ fn check_field_tys_sized<'tcx>(
 }
 
 impl<'tcx> crate::MirPass<'tcx> for StateTransform {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         let Some(old_yield_ty) = body.yield_ty() else {
             // This only applies to coroutines
             return;

compiler/rustc_mir_transform/src/coverage/mod.rs

@@ -19,7 +19,7 @@ use rustc_middle::mir::{
     self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator,
     TerminatorKind,
 };
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{BytePos, Pos, RelativeBytePos, Span, Symbol};
@@ -39,7 +39,12 @@ impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage {
         sess.instrument_coverage()
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
+    fn run_pass(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        _param_env: ParamEnv<'tcx>,
+        mir_body: &mut mir::Body<'tcx>,
+    ) {
         let mir_source = mir_body.source;
 
         // This pass runs after MIR promotion, but before promoted MIR starts to

compiler/rustc_mir_transform/src/ctfe_limit.rs

@@ -5,14 +5,14 @@ use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::{
     BasicBlock, BasicBlockData, Body, Statement, StatementKind, TerminatorKind,
 };
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::instrument;
 
 pub(super) struct CtfeLimit;
 
 impl<'tcx> crate::MirPass<'tcx> for CtfeLimit {
     #[instrument(skip(self, _tcx, body))]
-    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         let doms = body.basic_blocks.dominators();
         let indices: Vec<BasicBlock> = body
             .basic_blocks

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -13,7 +13,7 @@ use rustc_middle::mir::interpret::{InterpResult, Scalar};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
 use rustc_mir_dataflow::lattice::FlatSet;
 use rustc_mir_dataflow::value_analysis::{
     Map, PlaceIndex, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace,
@@ -36,7 +36,7 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
     }
 
     #[instrument(skip_all level = "debug")]
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         debug!(def_id = ?body.source.def_id());
         if tcx.sess.mir_opt_level() < 4 && body.basic_blocks.len() > BLOCK_LIMIT {
             debug!("aborted dataflow const prop due too many basic blocks");
@@ -57,7 +57,7 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
         let map = Map::new(tcx, body, place_limit);
 
         // Perform the actual dataflow analysis.
-        let analysis = ConstAnalysis::new(tcx, body, map);
+        let analysis = ConstAnalysis::new(tcx, param_env, body, map);
         let mut results = debug_span!("analyze")
             .in_scope(|| analysis.wrap().into_engine(tcx, body).iterate_to_fixpoint());
 
@@ -337,8 +337,12 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
 }
 
 impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
-    fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map<'tcx>) -> Self {
-        let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
+    fn new(
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        body: &'a Body<'tcx>,
+        map: Map<'tcx>,
+    ) -> Self {
         Self {
             map,
             tcx,

compiler/rustc_mir_transform/src/dead_store_elimination.rs

@@ -15,7 +15,7 @@
 use rustc_middle::bug;
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use rustc_mir_dataflow::Analysis;
 use rustc_mir_dataflow::debuginfo::debuginfo_locals;
 use rustc_mir_dataflow::impls::{
@@ -144,7 +144,7 @@ impl<'tcx> crate::MirPass<'tcx> for DeadStoreElimination {
         sess.mir_opt_level() >= 2
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         eliminate(tcx, body);
     }
 }

compiler/rustc_mir_transform/src/deduplicate_blocks.rs

@@ -8,7 +8,7 @@ use std::iter;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::debug;
 
 use super::simplify::simplify_cfg;
@@ -20,7 +20,7 @@ impl<'tcx> crate::MirPass<'tcx> for DeduplicateBlocks {
         sess.mir_opt_level() >= 4
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
         debug!("Running DeduplicateBlocks on `{:?}`", body.source);
         let duplicates = find_duplicates(body);
         let has_opts_to_apply = !duplicates.is_empty();