Post-mono MIR opts

Cargo.lock

@@ -3572,6 +3572,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_query_system",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -30,6 +30,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -14,17 +14,15 @@ use super::FunctionCx;
 use crate::traits::*;
 
 pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    fx: &FunctionCx<'a, 'tcx, Bx>,
-    traversal_order: &[mir::BasicBlock],
+    fx: &FunctionCx<'a, '_, 'tcx, Bx>,
 ) -> DenseBitSet<mir::Local> {
     let mir = fx.mir;
     let dominators = mir.basic_blocks.dominators();
     let locals = mir
         .local_decls
         .iter()
         .map(|decl| {
-            let ty = fx.monomorphize(decl.ty);
-            let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+            let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span);
             if layout.is_zst() { LocalKind::ZST } else { LocalKind::Unused }
         })
         .collect();
@@ -39,8 +37,7 @@ pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // If there exists a local definition that dominates all uses of that local,
     // the definition should be visited first. Traverse blocks in an order that
     // is a topological sort of dominance partial order.
-    for bb in traversal_order.iter().copied() {
-        let data = &mir.basic_blocks[bb];
+    for (bb, data) in traversal::reverse_postorder(mir) {
         analyzer.visit_basic_block_data(bb, data);
     }
 
@@ -65,9 +62,9 @@ enum LocalKind {
     SSA(DefLocation),
 }
 
-struct LocalAnalyzer<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
-    fx: &'a FunctionCx<'b, 'tcx, Bx>,
-    dominators: &'a Dominators<mir::BasicBlock>,
+struct LocalAnalyzer<'mir, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
+    fx: &'mir FunctionCx<'b, 'mir, 'tcx, Bx>,
+    dominators: &'mir Dominators<mir::BasicBlock>,
     locals: IndexVec<mir::Local, LocalKind>,
 }
 
@@ -80,8 +77,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
             LocalKind::ZST => {}
             LocalKind::Memory => {}
             LocalKind::Unused => {
-                let ty = fx.monomorphize(decl.ty);
-                let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+                let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span);
                 *kind =
                     if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) {
                         LocalKind::SSA(location)
@@ -117,10 +113,9 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
             );
             if is_consume {
                 let base_ty = place_base.ty(self.fx.mir, cx.tcx());
-                let base_ty = self.fx.monomorphize(base_ty);
 
                 // ZSTs don't require any actual memory access.
-                let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
+                let elem_ty = base_ty.projection_ty(cx.tcx(), elem).ty;
                 let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
                 if cx.spanned_layout_of(elem_ty, span).is_zst() {
                     return;
@@ -246,7 +241,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer
                 let kind = &mut self.locals[local];
                 if *kind != LocalKind::Memory {
                     let ty = self.fx.mir.local_decls[local].ty;
-                    let ty = self.fx.monomorphize(ty);
                     if self.fx.cx.type_needs_drop(ty) {
                         // Only need the place if we're actually dropping it.
                         *kind = LocalKind::Memory;

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -37,17 +37,17 @@ enum MergingSucc {
 
 /// Used by `FunctionCx::codegen_terminator` for emitting common patterns
 /// e.g., creating a basic block, calling a function, etc.
-struct TerminatorCodegenHelper<'tcx> {
+struct TerminatorCodegenHelper<'mir, 'tcx> {
     bb: mir::BasicBlock,
-    terminator: &'tcx mir::Terminator<'tcx>,
+    terminator: &'mir mir::Terminator<'tcx>,
 }
 
-impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
+impl<'a, 'tcx> TerminatorCodegenHelper<'_, 'tcx> {
     /// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
     /// either already previously cached, or newly created, by `landing_pad_for`.
     fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &'b mut FunctionCx<'a, '_, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
         let cleanup_kinds = fx.cleanup_kinds.as_ref()?;
         let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
@@ -73,7 +73,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// stuff in it or next to it.
     fn llbb_with_cleanup<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> Bx::BasicBlock {
         let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target);
@@ -97,7 +97,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn llbb_characteristics<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (bool, bool) {
         if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
@@ -122,7 +122,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         target: mir::BasicBlock,
         mergeable_succ: bool,
@@ -151,7 +151,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// return destination `destination` and the unwind action `unwind`.
     fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
         fn_ptr: Bx::Value,
@@ -274,7 +274,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// Generates inline assembly with optional `destination` and `unwind`.
     fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         template: &[InlineAsmTemplatePiece],
         operands: &[InlineAsmOperandRef<'tcx, Bx>],
@@ -341,9 +341,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 }
 
 /// Codegen implementations for some terminator variants.
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     /// Generates code for a `Resume` terminator.
-    fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) {
+    fn codegen_resume_terminator(
+        &mut self,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
+        bx: &mut Bx,
+    ) {
         if let Some(funclet) = helper.funclet(self) {
             bx.cleanup_ret(funclet, None);
         } else {
@@ -360,7 +364,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_switchint_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         discr: &mir::Operand<'tcx>,
         targets: &SwitchTargets,
@@ -567,7 +571,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[tracing::instrument(level = "trace", skip(self, helper, bx))]
     fn codegen_drop_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         source_info: &mir::SourceInfo,
         location: mir::Place<'tcx>,
@@ -576,7 +580,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         mergeable_succ: bool,
     ) -> MergingSucc {
         let ty = location.ty(self.mir, bx.tcx()).ty;
-        let ty = self.monomorphize(ty);
         let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
 
         if let ty::InstanceKind::DropGlue(_, None) = drop_fn.def {
@@ -665,7 +668,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_assert_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         cond: &mir::Operand<'tcx>,
@@ -755,7 +758,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_terminate_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         reason: UnwindTerminateReason,
@@ -785,7 +788,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// Returns `Some` if this is indeed a panic intrinsic and codegen is done.
     fn codegen_panic_intrinsic(
         &mut self,
-        helper: &TerminatorCodegenHelper<'tcx>,
+        helper: &TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         intrinsic: ty::IntrinsicDef,
         instance: Instance<'tcx>,
@@ -851,7 +854,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_call_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         func: &mir::Operand<'tcx>,
@@ -985,10 +988,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let sig = callee.layout.ty.fn_sig(bx.tcx());
 
         let extra_args = &args[sig.inputs().skip_binder().len()..];
-        let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| {
-            let op_ty = op_arg.node.ty(self.mir, bx.tcx());
-            self.monomorphize(op_ty)
-        }));
+        let extra_args = bx.tcx().mk_type_list_from_iter(
+            extra_args.iter().map(|op_arg| op_arg.node.ty(self.mir, bx.tcx())),
+        );
 
         let fn_abi = match instance {
             Some(instance) => bx.fn_abi_of_instance(instance, extra_args),
@@ -1153,7 +1155,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_asm_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         asm_macro: InlineAsmMacro,
         terminator: &mir::Terminator<'tcx>,
@@ -1196,8 +1198,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     InlineAsmOperandRef::Const { string }
                 }
                 mir::InlineAsmOperand::SymFn { ref value } => {
-                    let const_ = self.monomorphize(value.const_);
-                    if let ty::FnDef(def_id, args) = *const_.ty().kind() {
+                    if let ty::FnDef(def_id, args) = *value.const_.ty().kind() {
                         let instance = ty::Instance::resolve_for_fn_ptr(
                             bx.tcx(),
                             bx.typing_env(),
@@ -1287,7 +1288,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &mut self,
         bx: &mut Bx,
         bb: mir::BasicBlock,
-        terminator: &'tcx mir::Terminator<'tcx>,
+        terminator: &mir::Terminator<'tcx>,
     ) -> MergingSucc {
         debug!("codegen_terminator: {:?}", terminator);
 

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -9,21 +9,21 @@ use crate::errors;
 use crate::mir::operand::OperandRef;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub(crate) fn eval_mir_constant_to_operand(
         &self,
         bx: &mut Bx,
         constant: &mir::ConstOperand<'tcx>,
     ) -> OperandRef<'tcx, Bx::Value> {
         let val = self.eval_mir_constant(constant);
-        let ty = self.monomorphize(constant.ty());
-        OperandRef::from_const(bx, val, ty)
+        OperandRef::from_const(bx, val, constant.ty())
     }
 
     pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
         // `MirUsedCollector` visited all required_consts before codegen began, so if we got here
         // there can be no more constants that fail to evaluate.
-        self.monomorphize(constant.const_)
+        constant
+            .const_
             .eval(self.cx.tcx(), self.cx.typing_env(), constant.span)
             .expect("erroneous constant missed by mono item collection")
     }
@@ -38,7 +38,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &self,
         constant: &mir::ConstOperand<'tcx>,
     ) -> Result<Result<ty::ValTree<'tcx>, Ty<'tcx>>, ErrorHandled> {
-        let uv = match self.monomorphize(constant.const_) {
+        let uv = match constant.const_ {
             mir::Const::Unevaluated(uv, _) => uv.shrink(),
             mir::Const::Ty(_, c) => match c.kind() {
                 // A constant that came from a const generic but was then used as an argument to
@@ -56,7 +56,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             // real const generic, and get rid of this entire function.
             other => span_bug!(constant.span, "{other:#?}"),
         };
-        let uv = self.monomorphize(uv);
         self.cx.tcx().const_eval_resolve_for_typeck(self.cx.typing_env(), uv, constant.span)
     }
 
@@ -66,7 +65,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx: &Bx,
         constant: &mir::ConstOperand<'tcx>,
     ) -> (Bx::Value, Ty<'tcx>) {
-        let ty = self.monomorphize(constant.ty());
+        let ty = constant.ty();
         assert!(ty.is_simd());
         let field_ty = ty.simd_size_and_type(bx.tcx()).1;
 

compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs

@@ -4,14 +4,10 @@ use rustc_middle::mir::coverage::CoverageKind;
 use super::FunctionCx;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub(crate) fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) {
         // Determine the instance that coverage data was originally generated for.
-        let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) {
-            self.monomorphize(inlined)
-        } else {
-            self.instance
-        };
+        let instance = scope.inlined_instance(&self.mir.source_scopes).unwrap_or(self.instance);
 
         // Handle the coverage info in a backend-specific way.
         bx.add_coverage(instance, kind);

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -213,7 +213,7 @@ fn calculate_debuginfo_offset<
     DebugInfoOffset { direct_offset, indirect_offsets, result: place }
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) {
         bx.set_span(source_info.span);
         if let Some(dbg_loc) = self.dbg_loc(source_info) {
@@ -291,9 +291,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             // FIXME(eddyb) is this `+ 1` needed at all?
                             let kind = VariableKind::ArgumentVariable(arg_index + 1);
 
-                            let arg_ty = self.monomorphize(decl.ty);
-
-                            self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span)
+                            self.cx.create_dbg_var(name, decl.ty, dbg_scope, kind, span)
                         },
                     )
                 } else {
@@ -487,13 +485,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             };
 
             let var_ty = if let Some(ref fragment) = var.composite {
-                self.monomorphize(fragment.ty)
+                fragment.ty
             } else {
                 match var.value {
                     mir::VarDebugInfoContents::Place(place) => {
                         self.monomorphized_place_ty(place.as_ref())
                     }
-                    mir::VarDebugInfoContents::Const(c) => self.monomorphize(c.ty()),
+                    mir::VarDebugInfoContents::Const(c) => c.ty(),
                 }
             };
 

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -50,7 +50,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx.memset(dst, val, size, align, flags);
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     /// In the `Err` case, returns the instance that should be called instead.
     pub fn codegen_intrinsic_call(
         &mut self,