Auto merge of #143167 - Mark-Simulacrum:inline-mark-nounwind, r=<try>

Detect unwind-free functions in MIR

This adds analysis as part of inlining that will mark calls to functions with Unwind::Unreachable even if inlining is not possible, so long as their MIR indicates that they cannot unwind.

This would ideally persist into the codegen fn attrs, but that will take more work.

My guess is this implementation is not going to work in practice (too much overhead, not enough value?) but wanted to see what perf looked like.

Author: rust-bors[bot]

compiler/rustc_mir_transform/src/inline.rs

@@ -115,6 +115,7 @@ trait Inliner<'tcx> {
 
     /// Has the caller body been changed?
     fn changed(self) -> bool;
+    fn set_changed(&mut self);
 
     /// Should inlining happen for a given callee?
     fn should_inline_for_callee(&self, def_id: DefId) -> bool;
@@ -187,6 +188,10 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         self.changed
     }
 
+    fn set_changed(&mut self) {
+        self.changed = true;
+    }
+
     fn should_inline_for_callee(&self, def_id: DefId) -> bool {
         ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
     }
@@ -334,6 +339,10 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
         self.changed
     }
 
+    fn set_changed(&mut self) {
+        self.changed = true;
+    }
+
     fn should_inline_for_callee(&self, _: DefId) -> bool {
         true
     }
@@ -529,10 +538,35 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>(
         let span = trace_span!("process_blocks", %callsite.callee, ?bb);
         let _guard = span.enter();
 
-        match try_inlining(inliner, caller_body, &callsite) {
+        let mut unwind_unreachable = Err("did not reach analysis");
+        match try_inlining(inliner, caller_body, &callsite, &mut unwind_unreachable) {
             Err(reason) => {
                 debug!("not-inlined {} [{}]", callsite.callee, reason);
                 inliner.on_inline_failure(&callsite, reason);
+
+                match unwind_unreachable {
+                    Ok(()) => {
+                        if let Some(TerminatorKind::Call { unwind, .. }) =
+                            caller_body[callsite.block].terminator.as_mut().map(|v| &mut v.kind)
+                        {
+                            inliner.set_changed();
+                            tracing::info!("marked {} unwind unreachable", callsite.callee);
+                            *unwind = UnwindAction::Unreachable;
+                        } else {
+                            bug!(
+                                "unexpected terminator: {:?}",
+                                caller_body[callsite.block].terminator
+                            );
+                        }
+                    }
+                    Err(reason) => {
+                        tracing::info!(
+                            "not marking unwind unreachable {}: {}",
+                            callsite.callee,
+                            reason
+                        );
+                    }
+                }
             }
             Ok(new_blocks) => {
                 debug!("inlined {}", callsite.callee);
@@ -595,17 +629,69 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
     None
 }
 
+/// Ok indicates yes, Err(reason) otherwise.
+fn should_mark_nounwind<'tcx>(_tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> Result<(), &'static str> {
+    // Unwinds can only start at certain terminators.
+    for block in body.basic_blocks.iter() {
+        let unwind = match block.terminator().kind {
+            // These never unwind.
+            TerminatorKind::Goto { .. }
+            | TerminatorKind::SwitchInt { .. }
+            | TerminatorKind::UnwindTerminate(_)
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::CoroutineDrop
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::FalseUnwind { .. } => continue,
+
+            // Resume will *continue* unwinding, but if there's no other unwinding terminator it
+            // will never be reached.
+            TerminatorKind::UnwindResume => continue,
+
+            TerminatorKind::Yield { .. } => {
+                return Err("impl limitation: yield");
+            }
+
+            TerminatorKind::Drop { unwind, .. }
+            | TerminatorKind::Call { unwind, .. }
+            | TerminatorKind::Assert { unwind, .. } => unwind,
+
+            TerminatorKind::InlineAsm { .. } => return Err("inlineasm"),
+
+            TerminatorKind::TailCall { .. } => {
+                return Err("impl limitation: tail call");
+            }
+        };
+
+        match unwind {
+            UnwindAction::Continue => return Err("unwind: continue"),
+            // cannot unwind
+            UnwindAction::Unreachable => {}
+            // cannot unwind either -- will terminate instead
+            UnwindAction::Terminate(_) => {}
+            UnwindAction::Cleanup(_) => return Err("unwind: cleanup"),
+        }
+    }
+
+    // If we didn't find an unwinding terminator, the function cannot unwind.
+    Ok(())
+}
+
 /// Attempts to inline a callsite into the caller body. When successful returns basic blocks
 /// containing the inlined body. Otherwise returns an error describing why inlining didn't take
 /// place.
 fn try_inlining<'tcx, I: Inliner<'tcx>>(
     inliner: &I,
     caller_body: &mut Body<'tcx>,
     callsite: &CallSite<'tcx>,
+    unwind: &mut Result<(), &'static str>,
 ) -> Result<std::ops::Range<BasicBlock>, &'static str> {
     let tcx = inliner.tcx();
     check_mir_is_available(inliner, caller_body, callsite.callee)?;
 
+    let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
+    *unwind = should_mark_nounwind(tcx, callee_body);
+
     let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id());
     check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;
     check_codegen_attributes(inliner, callsite, callee_attrs)?;
@@ -622,7 +708,6 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
         }
     }
 
-    let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
     check_inline::is_inline_valid_on_body(tcx, callee_body)?;
     inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;
 

tests/codegen/mem-replace-big-type.rs

@@ -25,7 +25,7 @@ pub fn replace_big(dst: &mut Big, src: Big) -> Big {
 // CHECK-NOT: call void @llvm.memcpy
 
 // For a large type, we expect exactly three `memcpy`s
-// CHECK-LABEL: define internal void @{{.+}}mem{{.+}}replace{{.+}}(ptr
+// CHECK-LABEL: define void @{{.+}}mem{{.+}}replace{{.+}}(ptr
 // CHECK-SAME: sret([56 x i8]){{.+}}[[RESULT:%.+]], ptr{{.+}}%dest, ptr{{.+}}%src)
 // CHECK-NOT: call void @llvm.memcpy
 // CHECK: call void @llvm.memcpy.{{.+}}(ptr align 8 [[RESULT]], ptr align 8 %dest, i{{.*}} 56, i1 false)

tests/codegen/personality_lifetimes.rs

@@ -13,7 +13,9 @@ impl Drop for S {
 }
 
 #[inline(never)]
-fn might_unwind() {}
+fn might_unwind() {
+    panic!();
+}
 
 // CHECK-LABEL: @test
 #[no_mangle]

tests/mir-opt/inline/caller_with_trivial_bound.foo.Inline.panic-unwind.diff

@@ -10,7 +10,8 @@
 
       bb0: {
           StorageLive(_1);
-          _1 = bar::<T>() -> [return: bb1, unwind continue];
+-         _1 = bar::<T>() -> [return: bb1, unwind continue];
++         _1 = bar::<T>() -> [return: bb1, unwind unreachable];
       }
 
       bb1: {

tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff

@@ -56,7 +56,7 @@
 +         StorageLive(_17);
 +         StorageLive(_18);
 +         StorageLive(_19);
-+         _17 = <() as A>::call() -> [return: bb12, unwind continue];
++         _17 = <() as A>::call() -> [return: bb12, unwind unreachable];
       }
 
       bb1: {
@@ -124,11 +124,11 @@
 +     }
 + 
 +     bb12: {
-+         _18 = <() as A>::call() -> [return: bb13, unwind continue];
++         _18 = <() as A>::call() -> [return: bb13, unwind unreachable];
 +     }
 + 
 +     bb13: {
-+         _19 = <() as A>::call() -> [return: bb11, unwind continue];
++         _19 = <() as A>::call() -> [return: bb11, unwind unreachable];
       }
   }
 

tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir

@@ -12,7 +12,7 @@ fn main() -> () {
 
     bb0: {
         StorageLive(_1);
-        _1 = not_inlined() -> [return: bb1, unwind continue];
+        _1 = not_inlined() -> [return: bb1, unwind unreachable];
     }
 
     bb1: {
@@ -21,7 +21,7 @@ fn main() -> () {
         StorageLive(_3);
         StorageLive(_4);
         StorageLive(_5);
-        _3 = g() -> [return: bb3, unwind continue];
+        _3 = g() -> [return: bb3, unwind unreachable];
     }
 
     bb2: {
@@ -34,10 +34,10 @@ fn main() -> () {
     }
 
     bb3: {
-        _4 = g() -> [return: bb4, unwind continue];
+        _4 = g() -> [return: bb4, unwind unreachable];
     }
 
     bb4: {
-        _5 = g() -> [return: bb2, unwind continue];
+        _5 = g() -> [return: bb2, unwind unreachable];
     }
 }

tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff

@@ -20,7 +20,7 @@
 +         StorageLive(_1);
 +         StorageLive(_2);
 +         _1 = const inner::promoted[0];
-+         _0 = index() -> [return: bb1, unwind continue];
++         _0 = index() -> [return: bb1, unwind unreachable];
       }
 
       bb1: {

tests/mir-opt/inline/rustc_no_mir_inline.caller.Inline.panic-unwind.diff

@@ -7,7 +7,8 @@
 
       bb0: {
           StorageLive(_1);
-          _1 = callee() -> [return: bb1, unwind continue];
+-         _1 = callee() -> [return: bb1, unwind continue];
++         _1 = callee() -> [return: bb1, unwind unreachable];
       }
 
       bb1: {

tests/mir-opt/inline/rustc_no_mir_inline.caller.PreCodegen.after.panic-unwind.mir

@@ -5,7 +5,7 @@ fn caller() -> () {
     let _1: ();
 
     bb0: {
-        _1 = callee() -> [return: bb1, unwind continue];
+        _1 = callee() -> [return: bb1, unwind unreachable];
     }
 
     bb1: {

tests/mir-opt/inline/unsized_argument.caller.Inline.diff

@@ -13,11 +13,13 @@
           StorageLive(_3);
           _3 = move _1;
           _4 = copy ((_3.0: std::ptr::Unique<[i32]>).0: std::ptr::NonNull<[i32]>) as *const [i32] (Transmute);
-          _2 = callee(move (*_4)) -> [return: bb1, unwind: bb3];
+-         _2 = callee(move (*_4)) -> [return: bb1, unwind: bb3];
++         _2 = callee(move (*_4)) -> [return: bb1, unwind unreachable];
       }
 
       bb1: {
-          drop(_3) -> [return: bb2, unwind: bb4];
+-         drop(_3) -> [return: bb2, unwind: bb4];
++         drop(_3) -> [return: bb2, unwind: bb3];
       }
 
       bb2: {
@@ -28,10 +30,10 @@
       }
 
       bb3 (cleanup): {
-          drop(_3) -> [return: bb4, unwind terminate(cleanup)];
-      }
-  
-      bb4 (cleanup): {
+-         drop(_3) -> [return: bb4, unwind terminate(cleanup)];
+-     }
+- 
+-     bb4 (cleanup): {
           resume;
       }
   }

tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff

@@ -20,7 +20,7 @@
 +         nop;
 +         _6 = copy (*_1);
           _2 = copy (*_6);
-          _3 = unknown() -> [return: bb1, unwind continue];
+          _3 = unknown() -> [return: bb1, unwind unreachable];
       }
 
       bb1: {

tests/mir-opt/pre-codegen/deref_nested_borrows.src.PreCodegen.after.panic-unwind.mir

@@ -15,7 +15,7 @@ fn src(_1: &&u8) -> bool {
     bb0: {
         _2 = copy (*_1);
         _3 = copy (*_2);
-        _4 = unknown() -> [return: bb1, unwind continue];
+        _4 = unknown() -> [return: bb1, unwind unreachable];
     }
 
     bb1: {

tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff

@@ -76,9 +76,9 @@
           _7 = copy _9;
           StorageLive(_8);
 -         _8 = copy _1;
--         _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb5, unwind continue];
+-         _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb5, unwind unreachable];
 +         _8 = const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }};
-+         _6 = std::alloc::Global::alloc_impl(copy _9, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb5, unwind continue];
++         _6 = std::alloc::Global::alloc_impl(copy _9, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb5, unwind unreachable];
       }
 
       bb5: {

tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir

@@ -19,7 +19,7 @@ fn vec_move(_1: Vec<impl Sized>) -> () {
 
     bb0: {
         StorageLive(_2);
-        _2 = <Vec<impl Sized> as IntoIterator>::into_iter(move _1) -> [return: bb1, unwind continue];
+        _2 = <Vec<impl Sized> as IntoIterator>::into_iter(move _1) -> [return: bb1, unwind unreachable];
     }
 
     bb1: {
@@ -31,12 +31,12 @@ fn vec_move(_1: Vec<impl Sized>) -> () {
     bb2: {
         StorageLive(_5);
         _4 = &mut _3;
-        _5 = <std::vec::IntoIter<impl Sized> as Iterator>::next(move _4) -> [return: bb3, unwind: bb9];
+        _5 = <std::vec::IntoIter<impl Sized> as Iterator>::next(move _4) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
         _6 = discriminant(_5);
-        switchInt(move _6) -> [0: bb4, 1: bb6, otherwise: bb8];
+        switchInt(move _6) -> [0: bb4, 1: bb6, otherwise: bb10];
     }
 
     bb4: {
@@ -52,23 +52,23 @@ fn vec_move(_1: Vec<impl Sized>) -> () {
 
     bb6: {
         _7 = move ((_5 as Some).0: impl Sized);
-        _8 = opaque::<impl Sized>(move _7) -> [return: bb7, unwind: bb9];
+        _8 = opaque::<impl Sized>(move _7) -> [return: bb7, unwind: bb8];
     }
 
     bb7: {
         StorageDead(_5);
         goto -> bb2;
     }
 
-    bb8: {
-        unreachable;
+    bb8 (cleanup): {
+        drop(_3) -> [return: bb9, unwind terminate(cleanup)];
     }
 
     bb9 (cleanup): {
-        drop(_3) -> [return: bb10, unwind terminate(cleanup)];
+        resume;
     }
 
-    bb10 (cleanup): {
-        resume;
+    bb10: {
+        unreachable;
     }
 }