Move critical edge splitting to codegen.

Author: cjgillot

Cargo.lock

@@ -3463,6 +3463,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_monomorphize",
  "rustc_query_system",
  "rustc_serialize",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -27,6 +27,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_monomorphize = { path = "../rustc_monomorphize" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -7,6 +7,7 @@ use rustc_middle::mir::{traversal, UnwindTerminateReason};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_middle::{bug, mir, span_bug};
+use rustc_mir_transform::{add_call_guards, dump_mir, pass_manager};
 use rustc_target::abi::call::{FnAbi, PassMode};
 use tracing::{debug, instrument};
 
@@ -152,11 +153,23 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     let llfn = cx.get_fn(instance);
 
-    let mir = &instance.instantiate_mir_and_normalize_erasing_regions(
+    let mut mir = instance.instantiate_mir_and_normalize_erasing_regions(
         cx.tcx(),
         ty::ParamEnv::reveal_all(),
         ty::EarlyBinder::bind(cx.tcx().instance_mir(instance.def).clone()),
     );
+    pass_manager::run_passes(
+        cx.tcx(),
+        &mut mir,
+        &[
+            // Some cleanup necessary at least for LLVM and potentially other codegen backends.
+            &add_call_guards::CriticalCallEdges,
+            // Dump the end result for testing and debugging purposes.
+            &dump_mir::Marker("Monomorphic"),
+        ],
+        Some(mir::MirPhase::Runtime(mir::RuntimePhase::Monomorphic)),
+    );
+    let mir = &mir;
 
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
     debug!("fn_abi: {:?}", fn_abi);

compiler/rustc_middle/src/mir/mod.rs

@@ -159,6 +159,7 @@ impl RuntimePhase {
             "initial" => Self::Initial,
             "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
             "optimized" => Self::Optimized,
+            "monomorphic" => Self::Monomorphic,
             _ => bug!("Unknown runtime phase: '{}'", phase),
         }
     }

compiler/rustc_middle/src/mir/syntax.rs

@@ -99,6 +99,7 @@ impl MirPhase {
             MirPhase::Runtime(RuntimePhase::Initial) => "runtime",
             MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup",
             MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized",
+            MirPhase::Runtime(RuntimePhase::Monomorphic) => "runtime-monomorphic",
         }
     }
 
@@ -153,6 +154,8 @@ pub enum RuntimePhase {
     /// * [`ProjectionElem::Deref`] of `Box`
     PostCleanup = 1,
     Optimized = 2,
+    /// This corresponds to monomorphised MIR that is being processed by codegen.
+    Monomorphic = 3,
 }
 
 ///////////////////////////////////////////////////////////////////////////

compiler/rustc_mir_transform/src/lib.rs

@@ -38,12 +38,12 @@ use rustc_trait_selection::traits;
 use tracing::{debug, trace};
 
 #[macro_use]
-mod pass_manager;
+pub mod pass_manager;
 
 use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel};
 
 mod abort_unwinding_calls;
-mod add_call_guards;
+pub mod add_call_guards;
 mod add_moves_for_packed_drops;
 mod add_retag;
 mod add_subtyping_projections;
@@ -606,8 +606,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &multiple_return_terminators::MultipleReturnTerminators,
             &deduplicate_blocks::DeduplicateBlocks,
             &large_enums::EnumSizeOpt { discrepancy: 128 },
-            // Some cleanup necessary at least for LLVM and potentially other codegen backends.
-            &add_call_guards::CriticalCallEdges,
             // Cleanup for human readability, off by default.
             &prettify::ReorderBasicBlocks,
             &prettify::ReorderLocals,

compiler/rustc_mir_transform/src/pass_manager.rs

@@ -59,7 +59,7 @@ const fn c_name(name: &'static str) -> &'static str {
 /// A streamlined trait that you can implement to create a pass; the
 /// pass will be named after the type, and it will consist of a main
 /// loop that goes over each available MIR and applies `run_pass`.
-pub(super) trait MirPass<'tcx> {
+pub trait MirPass<'tcx> {
     fn name(&self) -> &'static str {
         // FIXME Simplify the implementation once more `str` methods get const-stable.
         // See copypaste in `MirLint`
@@ -160,7 +160,7 @@ pub(super) fn run_passes_no_validate<'tcx>(
 }
 
 /// The optional `phase_change` is applied after executing all the passes, if present
-pub(super) fn run_passes<'tcx>(
+pub fn run_passes<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     passes: &[&dyn MirPass<'tcx>],

compiler/rustc_mir_transform/src/validate.rs

@@ -391,7 +391,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                     // is used to simplify inserting code that should be executed along the return edge
                     // from the call. FIXME(tmiasko): Since this is a strictly code generation concern,
                     // the code generation should be responsible for handling it.
-                    if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                    if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Monomorphic)
                         && self.is_critical_call_edge(target, unwind)
                     {
                         self.fail(

tests/ui/mir/validate/critical-edge.rs

@@ -2,15 +2,14 @@
 //
 //@ build-fail
 //@ edition: 2021
-//@ compile-flags: --crate-type=lib
+//@ compile-flags: --crate-type=lib -Zvalidate-mir
 //@ failure-status: 101
 //@ dont-check-compiler-stderr
 //@ error-pattern: encountered critical edge in `Call` terminator
 #![feature(custom_mir, core_intrinsics)]
 use core::intrinsics::mir::*;
 
-#[custom_mir(dialect = "runtime", phase = "optimized")]
-#[inline(always)]
+#[custom_mir(dialect = "runtime", phase = "monomorphic")]
 pub fn f(a: u32) -> u32 {
     mir! {
         {