Auto merge of #113382 - lqd:test-mcp510, r=<try>

[perf] test MCP510

Author: rust-bors[bot]

compiler/rustc_borrowck/src/lib.rs

@@ -438,13 +438,24 @@ fn do_mir_borrowck<'tcx>(
 
     let (mut flow_analysis, flow_entry_states) =
         get_flow_results(tcx, body, &move_data, &borrow_set, &regioncx);
-    visit_results(
-        body,
-        traversal::reverse_postorder(body).map(|(bb, _)| bb),
-        &mut flow_analysis,
-        &flow_entry_states,
-        &mut mbcx,
-    );
+    if body.basic_blocks.is_cfg_cyclic() {
+        let _sccs = body.basic_blocks.sccs();
+        visit_results(
+            body,
+            traversal::reverse_postorder(body).map(|(bb, _)| bb),
+            &mut flow_analysis,
+            &flow_entry_states,
+            &mut mbcx,
+        );
+    } else {
+        visit_results(
+            body,
+            traversal::reverse_postorder(body).map(|(bb, _)| bb),
+            &mut flow_analysis,
+            &flow_entry_states,
+            &mut mbcx,
+        );
+    }
 
     mbcx.report_move_errors();
 

compiler/rustc_middle/src/mir/basic_blocks.rs

@@ -33,6 +33,93 @@ struct Cache {
     predecessors: OnceLock<Predecessors>,
     reverse_postorder: OnceLock<Vec<BasicBlock>>,
     dominators: OnceLock<Dominators<BasicBlock>>,
+    is_cyclic: OnceLock<bool>,
+    sccs: OnceLock<SccData>,
+}
+
+#[derive(Clone, Default, Debug)]
+pub struct SccData {
+    pub component_count: usize,
+
+    /// The SCC of each block.
+    pub components: IndexVec<BasicBlock, u32>,
+
+    /// The contents of each SCC: its blocks, in RPO.
+    pub sccs: Vec<SmallVec<[BasicBlock; 2]>>,
+}
+
+use std::collections::VecDeque;
+
+struct PearceRecursive {
+    r_index: IndexVec<BasicBlock, u32>,
+    stack: VecDeque<BasicBlock>,
+    index: u32,
+    c: u32,
+}
+
+impl PearceRecursive {
+    fn new(node_count: usize) -> Self {
+        assert!(node_count > 0); // only a non-empty graph is supported
+        // todo: assert node_count is within overflow limits
+        Self {
+            r_index: IndexVec::from_elem_n(0, node_count),
+            stack: VecDeque::new(),
+            index: 1,
+            c: node_count.try_into().unwrap(),
+            // c: node_count - 1,
+        }
+    }
+
+    fn compute_sccs(&mut self, blocks: &IndexVec<BasicBlock, BasicBlockData<'_>>) {
+        for v in blocks.indices() {
+            if self.r_index[v] == 0 {
+                self.visit(v, blocks);
+            }
+        }
+
+        // The SCC labels are from N - 1 to zero, remap them from 0 to the component count, to match
+        // their position in an array of SCCs.
+        let node_count: u32 = blocks.len().try_into().unwrap();
+        for scc_index in self.r_index.iter_mut() {
+            *scc_index = node_count - *scc_index - 1;
+        }
+
+        // Adjust the component index counter to the component count
+        self.c = node_count - self.c;
+    }
+
+    fn visit(&mut self, v: BasicBlock, blocks: &IndexVec<BasicBlock, BasicBlockData<'_>>) {
+        let mut root = true;
+        self.r_index[v] = self.index;
+        self.index += 1;
+
+        for w in blocks[v].terminator().successors() {
+            if self.r_index[w] == 0 {
+                self.visit(w, blocks);
+            }
+            if self.r_index[w] < self.r_index[v] {
+                self.r_index[v] = self.r_index[w];
+                root = false;
+            }
+        }
+
+        if root {
+            self.index -= 1;
+            self.c -= 1;
+
+            while let Some(&w) = self.stack.front()
+                && self.r_index[v] <= self.r_index[w]
+            {
+                self.stack.pop_front();
+                self.r_index[w] = self.c;
+                self.index -= 1;
+            }
+
+            self.r_index[v] = self.c;
+        } else {
+            self.stack.push_front(v);
+        }
+    }
 }
 
 impl<'tcx> BasicBlocks<'tcx> {
@@ -41,10 +128,35 @@ impl<'tcx> BasicBlocks<'tcx> {
         BasicBlocks { basic_blocks, cache: Cache::default() }
     }
 
+    /// Returns true if control-flow graph contains a cycle reachable from the `START_BLOCK`.
+    #[inline]
+    pub fn is_cfg_cyclic(&self) -> bool {
+        *self.cache.is_cyclic.get_or_init(|| graph::is_cyclic(self))
+    }
+
+    #[inline]
     pub fn dominators(&self) -> &Dominators<BasicBlock> {
         self.cache.dominators.get_or_init(|| dominators(self))
     }
 
+    #[inline]
+    pub fn sccs(&self) -> &SccData {
+        self.cache.sccs.get_or_init(|| {
+            let block_count = self.basic_blocks.len();
+
+            let mut pearce = PearceRecursive::new(block_count);
+            pearce.compute_sccs(&self.basic_blocks);
+            let component_count = pearce.c as usize;
+
+            let mut sccs = vec![smallvec::SmallVec::new(); component_count];
+            for &block in self.reverse_postorder().iter() {
+                let scc = pearce.r_index[block] as usize;
+                sccs[scc].push(block);
+            }
+            SccData { component_count, components: pearce.r_index, sccs }
+        })
+    }
+
     /// Returns predecessors for each basic block.
     #[inline]
     pub fn predecessors(&self) -> &Predecessors {

compiler/rustc_mir_dataflow/src/framework/mod.rs

@@ -229,6 +229,20 @@ pub trait Analysis<'tcx> {
         unreachable!();
     }
 
+    #[inline]
+    fn iterate_to_fixpoint<'mir>(
+        self,
+        tcx: TyCtxt<'tcx>,
+        body: &'mir mir::Body<'tcx>,
+        pass_name: Option<&'static str>,
+    ) -> AnalysisAndResults<'tcx, Self>
+    where
+        Self: Sized,
+        Self::Domain: DebugWithContext<Self>,
+    {
+        self.iterate_to_fixpoint_per_block(tcx, body, pass_name)
+    }
+
     /* Extension methods */
 
     /// Finds the fixpoint for this dataflow problem.
@@ -244,7 +258,7 @@ pub trait Analysis<'tcx> {
     /// dataflow analysis. Some analyses are run multiple times in the compilation pipeline.
     /// Without a `pass_name` to differentiates them, only the results for the latest run will be
     /// saved.
-    fn iterate_to_fixpoint<'mir>(
+    fn iterate_to_fixpoint_per_block<'mir>(
         mut self,
         tcx: TyCtxt<'tcx>,
         body: &'mir mir::Body<'tcx>,
@@ -308,6 +322,92 @@ pub trait Analysis<'tcx> {
 
         AnalysisAndResults { analysis: self, results }
     }
+
+    fn iterate_to_fixpoint_per_scc<'mir>(
+        mut self,
+        _tcx: TyCtxt<'tcx>,
+        body: &'mir mir::Body<'tcx>,
+        _pass_name: Option<&'static str>,
+    ) -> AnalysisAndResults<'tcx, Self>
+    where
+        Self: Sized,
+        Self::Domain: DebugWithContext<Self>,
+    {
+        assert!(Self::Direction::IS_FORWARD);
+
+        let sccs = body.basic_blocks.sccs();
+
+        struct VecQueue<T: Idx> {
+            queue: Vec<T>,
+            set: DenseBitSet<T>,
+        }
+
+        impl<T: Idx> VecQueue<T> {
+            #[inline]
+            fn with_none(len: usize) -> Self {
+                VecQueue { queue: Vec::with_capacity(len), set: DenseBitSet::new_empty(len) }
+            }
+
+            #[inline]
+            fn insert(&mut self, element: T) {
+                if self.set.insert(element) {
+                    self.queue.push(element);
+                }
+            }
+        }
+
+        let mut scc_queue = VecQueue::with_none(sccs.component_count);
+        for &bb in body.basic_blocks.reverse_postorder().iter() {
+            // let scc = sccs.components[bb.as_usize()];
+            let scc = sccs.components[bb];
+            scc_queue.insert(scc);
+        }
+        // assert_eq!(scc_queue.queue, sccs.queue);
+
+        let mut results = IndexVec::from_fn_n(|_| self.bottom_value(body), body.basic_blocks.len());
+        self.initialize_start_block(body, &mut results[mir::START_BLOCK]);
+
+        // Worklist for per-SCC iterations
+        let mut dirty_queue: WorkQueue<BasicBlock> = WorkQueue::with_none(body.basic_blocks.len());
+
+        let mut state = self.bottom_value(body);
+
+        for &scc in &scc_queue.queue {
+            // les blocks doivent être ajoutés en RPO
+            // for block in sccs.blocks_in_rpo(scc as usize) {
+            for block in sccs.sccs[scc as usize].iter().copied() {
+                dirty_queue.insert(block);
+            }
+
+            while let Some(bb) = dirty_queue.pop() {
+                // Set the state to the entry state of the block. This is equivalent to `state =
+                // results[bb].clone()`, but it saves an allocation, thus improving compile times.
+                state.clone_from(&results[bb]);
+
+                Self::Direction::apply_effects_in_block(
+                    &mut self,
+                    body,
+                    &mut state,
+                    bb,
+                    &body[bb],
+                    |target: BasicBlock, state: &Self::Domain| {
+                        let set_changed = results[target].join(state);
+                        // let target_scc = sccs.components[target.as_usize()];
+                        let target_scc = sccs.components[target];
+                        if set_changed && target_scc == scc {
+                            // The target block is in the SCC we're currently processing, and we
+                            // want to process this block until fixpoint. Otherwise, the target
+                            // block is in a successor SCC and it will be processed when that SCC is
+                            // encountered later.
+                            dirty_queue.insert(target);
+                        }
+                    },
+                );
+            }
+        }
+
+        AnalysisAndResults { analysis: self, results }
+    }
 }
 
 /// The legal operations for a transfer function in a gen/kill problem.