Skip to content

Commit

Permalink
Auto merge of rust-lang#131650 - saethlin:post-mono-mir-opts, r=<try>
Browse files Browse the repository at this point in the history
Add post-mono MIR passes to make mono-reachable analysis more accurate

As of rust-lang#131650 (comment) I believe most of the incr overhead comes from re-computing, re-encoding, and loading a lot more MIR when all we're actually doing is traversing through it. I think that can be addressed by caching a query that looks up the mentioned/used items for an Instance.

I think the full-build regressions are pretty much just the expense of cloning, then monomorphizing, then caching the MIR.
  • Loading branch information
bors committed Oct 25, 2024
2 parents 45089ec + 4d85f3e commit ff70274
Show file tree
Hide file tree
Showing 85 changed files with 490 additions and 530 deletions.
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_cranelift/src/base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ pub(crate) fn codegen_fn<'tcx>(
let symbol_name = tcx.symbol_name(instance).name.to_string();
let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);

let mir = tcx.instance_mir(instance.def);
let mir = tcx.codegen_mir(instance);
let _mir_guard = crate::PrintOnPanic(|| {
let mut buf = Vec::new();
with_no_trimmed_paths!({
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_ssa/src/base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
// release builds.
info!("codegen_instance({})", instance);

mir::codegen_mir::<Bx>(cx, instance);
mir::lower_mir::<Bx>(cx, instance);
}

/// Creates the `main` function which will initialize the rust runtime and call
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_ssa/src/mir/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,15 +159,15 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
///////////////////////////////////////////////////////////////////////////

#[instrument(level = "debug", skip(cx))]
pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
pub fn lower_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cx: &'a Bx::CodegenCx,
instance: Instance<'tcx>,
) {
assert!(!instance.args.has_infer());

let llfn = cx.get_fn(instance);

let mir = cx.tcx().instance_mir(instance.def);
let mir = cx.tcx().codegen_mir(instance);

let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
debug!("fn_abi: {:?}", fn_abi);
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_middle/src/mir/mono.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ impl<'tcx> MonoItem<'tcx> {
InstanceKind::Item(..)
| InstanceKind::DropGlue(..)
| InstanceKind::AsyncDropGlueCtorShim(..) => {
let mir = tcx.instance_mir(instance.def);
let mir = tcx.codegen_mir(instance);
mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum()
}
// Other compiler-generated shims size estimate: 1
Expand Down
11 changes: 9 additions & 2 deletions compiler/rustc_middle/src/query/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -555,14 +555,21 @@ rustc_queries! {
desc { |tcx| "verify auto trait bounds for coroutine interior type `{}`", tcx.def_path_str(key) }
}

/// MIR after our optimization passes have run. This is MIR that is ready
/// for codegen. This is also the only query that can fetch non-local MIR, at present.
/// Polymorphic MIR after our pre-mono optimization passes have run. This is the MIR that
/// crates export.
query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
desc { |tcx| "optimizing MIR for `{}`", tcx.def_path_str(key) }
cache_on_disk_if { key.is_local() }
separate_provide_extern
}

/// MIR for a specific Instance ready for codegen. This is `optimized_mir` but monomorphized
/// and with extra transforms applied.
query build_codegen_mir(key: ty::Instance<'tcx>) -> &'tcx mir::Body<'tcx> {
desc { |tcx| "finalizing codegen MIR for `{}`", tcx.def_path_str_with_args(key.def_id(), key.args) }
arena_cache
}

/// Checks for the nearest `#[coverage(off)]` or `#[coverage(on)]` on
/// this def and any enclosing defs, up to the crate root.
///
Expand Down
8 changes: 8 additions & 0 deletions compiler/rustc_middle/src/ty/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1719,6 +1719,14 @@ impl<'tcx> TyCtxt<'tcx> {
}
}

pub fn codegen_mir(self, instance: ty::Instance<'tcx>) -> &'tcx Body<'tcx> {
if self.sess.opts.incremental == None {
self.build_codegen_mir(instance)
} else {
self.instance_mir(instance.def)
}
}

/// Returns the possibly-auto-generated MIR of a [`ty::InstanceKind`].
#[instrument(skip(self), level = "debug")]
pub fn instance_mir(self, instance: ty::InstanceKind<'tcx>) -> &'tcx Body<'tcx> {
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use rustc_ast::InlineAsmOptions;
use rustc_middle::mir::*;
use rustc_middle::span_bug;
use rustc_middle::ty::{self, TyCtxt, layout};
use rustc_middle::ty::{self, ParamEnv, TyCtxt, layout};
use rustc_target::spec::PanicStrategy;
use rustc_target::spec::abi::Abi;

Expand All @@ -23,7 +23,7 @@ use rustc_target::spec::abi::Abi;
pub(super) struct AbortUnwindingCalls;

impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
let def_id = body.source.def_id();
let kind = tcx.def_kind(def_id);

Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/add_call_guards.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use rustc_index::{Idx, IndexVec};
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use tracing::debug;

#[derive(PartialEq)]
Expand Down Expand Up @@ -31,7 +31,7 @@ pub(super) use self::AddCallGuards::*;
*/

impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
let mut pred_count: IndexVec<_, _> =
body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect();
pred_count[START_BLOCK] += 1;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use rustc_middle::mir::patch::MirPatch;
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use tracing::debug;

use crate::util;
Expand Down Expand Up @@ -38,12 +38,10 @@ use crate::util;
pub(super) struct AddMovesForPackedDrops;

impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
debug!("add_moves_for_packed_drops({:?} @ {:?})", body.source, body.span);

let def_id = body.source.def_id();
let mut patch = MirPatch::new(body);
let param_env = tcx.param_env(def_id);

for (bb, data) in body.basic_blocks.iter_enumerated() {
let loc = Location { block: bb, statement_index: data.statements.len() };
Expand Down
6 changes: 3 additions & 3 deletions compiler/rustc_mir_transform/src/add_retag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

use rustc_hir::LangItem;
use rustc_middle::mir::*;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};

pub(super) struct AddRetag;

Expand Down Expand Up @@ -53,9 +53,9 @@ impl<'tcx> crate::MirPass<'tcx> for AddRetag {
sess.opts.unstable_opts.mir_emit_retag
}

fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
// We need an `AllCallEdges` pass before we can do any work.
super::add_call_guards::AllCallEdges.run_pass(tcx, body);
super::add_call_guards::AllCallEdges.run_pass(tcx, param_env, body);

let basic_blocks = body.basic_blocks.as_mut();
let local_decls = &body.local_decls;
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/add_subtyping_projections.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use rustc_middle::mir::patch::MirPatch;
use rustc_middle::mir::visit::MutVisitor;
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};

pub(super) struct Subtyper;

Expand Down Expand Up @@ -52,7 +52,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for SubTypeChecker<'a, 'tcx> {
// let temp: rval_ty = rval;
// let place: place_ty = temp as place_ty;
impl<'tcx> crate::MirPass<'tcx> for Subtyper {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
let patch = MirPatch::new(body);
let mut checker = SubTypeChecker { tcx, patcher: patch, local_decls: &body.local_decls };

Expand Down
3 changes: 1 addition & 2 deletions compiler/rustc_mir_transform/src/check_alignment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment {
sess.ub_checks()
}

fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
// This pass emits new panics. If for whatever reason we do not have a panic
// implementation, running this pass may cause otherwise-valid code to not compile.
if tcx.lang_items().get(LangItem::PanicImpl).is_none() {
Expand All @@ -27,7 +27,6 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment {

let basic_blocks = body.basic_blocks.as_mut();
let local_decls = &mut body.local_decls;
let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());

// This pass inserts new blocks. Each insertion changes the Location for all
// statements/blocks after. Iterating or visiting the MIR in order would require updating
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@

use rustc_middle::mir::coverage::CoverageKind;
use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, TerminatorKind};
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::{ParamEnv, TyCtxt};

pub(super) struct CleanupPostBorrowck;

impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
for basic_block in body.basic_blocks.as_mut() {
for statement in basic_block.statements.iter_mut() {
match statement.kind {
Expand Down
5 changes: 2 additions & 3 deletions compiler/rustc_mir_transform/src/copy_prop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use rustc_index::IndexSlice;
use rustc_index::bit_set::BitSet;
use rustc_middle::mir::visit::*;
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use tracing::{debug, instrument};

use crate::ssa::SsaLocals;
Expand All @@ -25,10 +25,9 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp {
}

#[instrument(level = "trace", skip(self, tcx, body))]
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
debug!(def_id = ?body.source.def_id());

let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
let ssa = SsaLocals::new(tcx, body, param_env);

let fully_moved = fully_moved_locals(&ssa, body);
Expand Down
12 changes: 9 additions & 3 deletions compiler/rustc_mir_transform/src/coroutine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ use rustc_index::{Idx, IndexVec};
use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
use rustc_middle::mir::*;
use rustc_middle::ty::{
self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt,
self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, ParamEnv, Ty, TyCtxt,
};
use rustc_middle::{bug, span_bug};
use rustc_mir_dataflow::Analysis;
Expand Down Expand Up @@ -1361,7 +1361,13 @@ fn create_coroutine_resume_function<'tcx>(
// unrelated code from the drop part of the function
simplify::remove_dead_blocks(body);

pm::run_passes_no_validate(tcx, body, &[&abort_unwinding_calls::AbortUnwindingCalls], None);
pm::run_passes_no_validate(
tcx,
tcx.param_env_reveal_all_normalized(body.source.def_id()),
body,
&[&abort_unwinding_calls::AbortUnwindingCalls],
None,
);

dump_mir(tcx, false, "coroutine_resume", &0, body, |_, _| Ok(()));
}
Expand Down Expand Up @@ -1526,7 +1532,7 @@ fn check_field_tys_sized<'tcx>(
}

impl<'tcx> crate::MirPass<'tcx> for StateTransform {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
let Some(old_yield_ty) = body.yield_ty() else {
// This only applies to coroutines
return;
Expand Down
9 changes: 7 additions & 2 deletions compiler/rustc_mir_transform/src/coverage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use rustc_middle::mir::{
self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator,
TerminatorKind,
};
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use rustc_span::def_id::LocalDefId;
use rustc_span::source_map::SourceMap;
use rustc_span::{BytePos, Pos, RelativeBytePos, Span, Symbol};
Expand All @@ -39,7 +39,12 @@ impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage {
sess.instrument_coverage()
}

fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
fn run_pass(
&self,
tcx: TyCtxt<'tcx>,
_param_env: ParamEnv<'tcx>,
mir_body: &mut mir::Body<'tcx>,
) {
let mir_source = mir_body.source;

// This pass runs after MIR promotion, but before promoted MIR starts to
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/ctfe_limit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ use rustc_data_structures::graph::dominators::Dominators;
use rustc_middle::mir::{
BasicBlock, BasicBlockData, Body, Statement, StatementKind, TerminatorKind,
};
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use tracing::instrument;

pub(super) struct CtfeLimit;

impl<'tcx> crate::MirPass<'tcx> for CtfeLimit {
#[instrument(skip(self, _tcx, body))]
fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
let doms = body.basic_blocks.dominators();
let indices: Vec<BasicBlock> = body
.basic_blocks
Expand Down
14 changes: 9 additions & 5 deletions compiler/rustc_mir_transform/src/dataflow_const_prop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use rustc_middle::mir::interpret::{InterpResult, Scalar};
use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
use rustc_middle::mir::*;
use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
use rustc_mir_dataflow::lattice::FlatSet;
use rustc_mir_dataflow::value_analysis::{
Map, PlaceIndex, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace,
Expand All @@ -36,7 +36,7 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
}

#[instrument(skip_all level = "debug")]
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
debug!(def_id = ?body.source.def_id());
if tcx.sess.mir_opt_level() < 4 && body.basic_blocks.len() > BLOCK_LIMIT {
debug!("aborted dataflow const prop due too many basic blocks");
Expand All @@ -57,7 +57,7 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
let map = Map::new(tcx, body, place_limit);

// Perform the actual dataflow analysis.
let analysis = ConstAnalysis::new(tcx, body, map);
let analysis = ConstAnalysis::new(tcx, param_env, body, map);
let mut results = debug_span!("analyze")
.in_scope(|| analysis.wrap().into_engine(tcx, body).iterate_to_fixpoint());

Expand Down Expand Up @@ -337,8 +337,12 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
}

impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map<'tcx>) -> Self {
let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
fn new(
tcx: TyCtxt<'tcx>,
param_env: ParamEnv<'tcx>,
body: &'a Body<'tcx>,
map: Map<'tcx>,
) -> Self {
Self {
map,
tcx,
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/dead_store_elimination.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
use rustc_middle::bug;
use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use rustc_mir_dataflow::Analysis;
use rustc_mir_dataflow::debuginfo::debuginfo_locals;
use rustc_mir_dataflow::impls::{
Expand Down Expand Up @@ -144,7 +144,7 @@ impl<'tcx> crate::MirPass<'tcx> for DeadStoreElimination {
sess.mir_opt_level() >= 2
}

fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
eliminate(tcx, body);
}
}
4 changes: 2 additions & 2 deletions compiler/rustc_mir_transform/src/deduplicate_blocks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::iter;
use rustc_data_structures::fx::FxHashMap;
use rustc_middle::mir::visit::MutVisitor;
use rustc_middle::mir::*;
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::{ParamEnv, TyCtxt};
use tracing::debug;

use super::simplify::simplify_cfg;
Expand All @@ -20,7 +20,7 @@ impl<'tcx> crate::MirPass<'tcx> for DeduplicateBlocks {
sess.mir_opt_level() >= 4
}

fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
fn run_pass(&self, tcx: TyCtxt<'tcx>, _param_env: ParamEnv<'tcx>, body: &mut Body<'tcx>) {
debug!("Running DeduplicateBlocks on `{:?}`", body.source);
let duplicates = find_duplicates(body);
let has_opts_to_apply = !duplicates.is_empty();
Expand Down
Loading

0 comments on commit ff70274

Please sign in to comment.