Auto merge of #165101 - stephenduong1004:propagate-cold, r=<try>

Propagate block coldness across unwind edges and mark cold callsites

Author: bors

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1578,6 +1578,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
         attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
     }
+
+    fn apply_attrs_to_cold_callsite(&mut self, llret: &'ll Value) {
+        let cold_attr = llvm::AttributeKind::Cold.create_attr(self.llcx);
+        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_attr]);
+    }
 }
 
 impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -266,6 +266,16 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             return MergingSucc::False;
         }
 
+        let is_cleanup = fx.mir[self.bb].is_cleanup;
+        let is_cold = fx.cold_blocks.contains(self.bb);
+        let apply_callsite_attrs = |bx: &mut Bx, call_llval| {
+            if is_cleanup {
+                bx.apply_attrs_to_cleanup_callsite(call_llval);
+            } else if is_cold {
+                bx.apply_attrs_to_cold_callsite(call_llval);
+            }
+        };
+
         if let Some(unwind_block) = unwind_block {
             let ret_llbb = if let Some((_, target)) = destination {
                 self.llbb_with_cleanup(fx, target)
@@ -284,9 +294,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 self.funclet(fx),
                 instance,
             );
-            if fx.mir[self.bb].is_cleanup {
-                bx.apply_attrs_to_cleanup_callsite(invokeret);
-            }
+            apply_callsite_attrs(bx, invokeret);
 
             if let Some((ret_dest, target)) = destination {
                 bx.switch_to_block(fx.llbb(target));
@@ -314,9 +322,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 self.funclet(fx),
                 instance,
             );
-            if fx.mir[self.bb].is_cleanup {
-                bx.apply_attrs_to_cleanup_callsite(llret);
-            }
+            apply_callsite_attrs(bx, llret);
 
             if let Some((ret_dest, target)) = destination {
                 for &(tmp, size) in lifetime_ends_after_call {

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -675,6 +675,9 @@ fn find_cold_blocks<'tcx>(
 
     // Traverse all basic blocks from end of the function to the start.
     for (bb, bb_data) in traversal::postorder(mir) {
+        if bb_data.is_cleanup {
+            continue;
+        }
         let terminator = bb_data.terminator();
 
         match terminator.kind {
@@ -698,8 +701,8 @@ fn find_cold_blocks<'tcx>(
             _ => {}
         }
 
-        // If all successors of a BB are cold and there's at least one of them, mark this BB as cold
-        let mut succ = terminator.successors();
+        // If all non-cleanup successors of a BB are cold and there's at least one of them, mark this BB as cold.
+        let mut succ = terminator.successors().filter(|&s| !mir.basic_blocks[s].is_cleanup);
         if let Some(first) = succ.next()
             && cold_blocks.contains(first)
             && succ.all(|s| cold_blocks.contains(s))

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -690,4 +690,8 @@ pub trait BuilderMethods<'a, 'tcx>:
     fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
 
     fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value);
+
+    fn apply_attrs_to_cold_callsite(&mut self, llret: Self::Value) {
+        self.apply_attrs_to_cleanup_callsite(llret);
+    }
 }