Give spilled bundles a register in the loops where they are used

src/fuzzing/func.rs

@@ -560,13 +560,31 @@ impl Func {
                         // is less than the number of registers in any single
                         // class, so the resulting problem is always
                         // allocatable.
-                        for _ in 0..u.int_in_range(opts.num_clobbers_per_inst.clone())? {
-                            let reg = u.int_in_range(0..=30)?;
-                            let preg = PReg::new(reg, RegClass::arbitrary(u)?);
-                            if operands.iter().any(has_fixed_def_with(preg)) {
-                                continue;
+                        if bool::arbitrary(u)? {
+                            // Or clobber nearly every register of one class, as a
+                            // call does, so that values live across this
+                            // instruction must be spilled. One register per
+                            // operand of that class stays free, at the top so
+                            // that every preferred register is clobbered, which
+                            // keeps the instruction itself allocatable.
+                            let class = RegClass::arbitrary(u)?;
+                            let free = operands.iter().filter(|op| op.class() == class).count();
+                            for reg in 0..32 - free {
+                                let preg = PReg::new(reg, class);
+                                if operands.iter().any(has_fixed_def_with(preg)) {
+                                    continue;
+                                }
+                                clobbers.push(preg);
+                            }
+                        } else {
+                            for _ in 0..u.int_in_range(opts.num_clobbers_per_inst.clone())? {
+                                let reg = u.int_in_range(0..=30)?;
+                                let preg = PReg::new(reg, RegClass::arbitrary(u)?);
+                                if operands.iter().any(has_fixed_def_with(preg)) {
+                                    continue;
+                                }
+                                clobbers.push(preg);
                             }
-                            clobbers.push(preg);
                         }
                     }
                 } else if opts.clobbers && bool::arbitrary(u)? {

src/ion/data_structures.rs

@@ -130,6 +130,9 @@ pub struct LiveRange {
 #[repr(u32)]
 pub enum LiveRangeFlag {
     StartsAtDef = 1,
+    /// The value is also in its spillset's spill slot throughout the range,
+    /// so moving it there at the end of the range is redundant.
+    InSpillSlot = 2,
 }
 
 impl LiveRange {
@@ -567,6 +570,21 @@ pub struct Ctx {
     pub(crate) scratch_removed_lrs_vregs: FxHashSet<VRegIndex>,
     pub(crate) scratch_workqueue_set: FxHashSet<Block>,
 
+    // For each block, the first and last block of the innermost loop
+    // containing it (see `try_allocating_regs_in_loops`).
+    pub(crate) scratch_loop_runs: Vec<(u32, u32)>,
+    // Per register class, the sorted points at which every allocatable
+    // register of that class is clobbered, and those at which every
+    // preferred one is (calls, when the preferred ones are caller-saved).
+    pub(crate) scratch_full_clobbers: [Vec<ProgPoint>; 3],
+    pub(crate) scratch_call_points: [Vec<ProgPoint>; 3],
+    // Candidate loop regions: (depth, first block, last block).
+    pub(crate) scratch_loop_regions: Vec<(u32, u32, u32)>,
+    // The uses of a spilled bundle's vregs, sorted by vreg and position.
+    pub(crate) scratch_loop_uses: Vec<(VRegIndex, ProgPoint)>,
+    // Whether any range has `LiveRangeFlag::InSpillSlot` in this function.
+    pub(crate) any_in_spill_slot: bool,
+
     pub(crate) scratch_bump: UnsafeSendSync<Bump>,
 }
 
@@ -629,6 +647,12 @@ impl Default for Ctx {
             scratch_removed_lrs: Default::default(),
             scratch_removed_lrs_vregs: Default::default(),
             scratch_workqueue_set: Default::default(),
+            scratch_loop_runs: Default::default(),
+            scratch_full_clobbers: Default::default(),
+            scratch_call_points: Default::default(),
+            scratch_loop_regions: Default::default(),
+            scratch_loop_uses: Default::default(),
+            any_in_spill_slot: false,
         }
     }
 }
@@ -973,6 +997,9 @@ pub struct Stats {
     pub spill_bundle_count: usize,
     pub spill_bundle_reg_probes: usize,
     pub spill_bundle_reg_success: usize,
+    pub spill_bundle_loop_region_probes: usize,
+    pub spill_bundle_loop_region_success: usize,
+    pub spill_slot_stores_elided: usize,
     pub blockparam_ins_count: usize,
     pub blockparam_outs_count: usize,
     pub halfmoves_count: usize,

src/ion/moves.rs

@@ -73,6 +73,22 @@ impl<'a, F: Function> Env<'a, F> {
         }
     }
 
+    /// The spill slot that also holds `vreg` at the end of `block`, as marked
+    /// by the loop-region pass, if any. Only called when a value in a
+    /// register would be moved to the stack there.
+    fn spill_slot_holding(&self, vreg: VRegIndex, block: Block) -> Option<Allocation> {
+        if !self.ctx.any_in_spill_slot {
+            return None;
+        }
+        let exit = self.ctx.cfginfo.block_exit[block.index()];
+        let entry = self.ctx.vregs[vreg].ranges.iter().find(|entry| {
+            entry.range.contains_point(exit)
+                && self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::InSpillSlot)
+        })?;
+        let spillset = self.ctx.bundles[self.ctx.ranges[entry.index].bundle].spillset;
+        Some(self.ctx.spillslots[self.ctx.spillsets[spillset].slot.index()].alloc)
+    }
+
     pub fn apply_allocations_and_insert_moves(&mut self) -> InsertedMoves {
         trace!("apply_allocations_and_insert_moves");
         trace!("blockparam_ins: {:?}", self.blockparam_ins);
@@ -631,6 +647,21 @@ impl<'a, F: Function> Env<'a, F> {
                 trace!("processing inter-block moves for {}", vreg);
                 for dest in inter_block_dests.drain(..) {
                     let src = inter_block_sources[&dest.from];
+                    if src.is_reg()
+                        && dest.alloc.is_stack()
+                        && self.spill_slot_holding(VRegIndex::new(vreg.vreg()), dest.from)
+                            == Some(dest.alloc)
+                    {
+                        trace!(
+                            " -> {} already holds {} between {:?} and {:?}",
+                            dest.alloc,
+                            vreg,
+                            dest.from,
+                            dest.to
+                        );
+                        self.output.stats.spill_slot_stores_elided += 1;
+                        continue;
+                    }
 
                     trace!(
                         " -> moving from {} to {} between {:?} and {:?}",
@@ -653,9 +684,33 @@ impl<'a, F: Function> Env<'a, F> {
             self.output.stats.halfmoves_count += block_param_dests.len();
 
             trace!("processing block-param moves");
+            let mut block_param_args = FxHashMap::<BlockparamSourceKey, VRegIndex>::default();
             for dest in block_param_dests {
                 let src = dest.source();
                 let src_alloc = block_param_sources.get(&src).unwrap();
+                if self.ctx.any_in_spill_slot && src_alloc.is_reg() && dest.alloc.is_stack() {
+                    // Which value is passed, for the few moves that could
+                    // store one already in its slot.
+                    if block_param_args.is_empty() {
+                        block_param_args.extend(self.blockparam_outs.iter().map(|out| {
+                            (
+                                BlockparamSourceKey::new(out.from_block, out.to_vreg),
+                                out.from_vreg,
+                            )
+                        }));
+                    }
+                    let from_vreg = block_param_args[&src];
+                    if self.spill_slot_holding(from_vreg, dest.from_block) == Some(dest.alloc) {
+                        trace!(
+                            " -> {} already holds the value of v{} from block{}",
+                            dest.alloc,
+                            dest.to_vreg.index(),
+                            dest.from_block.index()
+                        );
+                        self.output.stats.spill_slot_stores_elided += 1;
+                        continue;
+                    }
+                }
                 let (pos, prio) = choose_move_location(self, dest.from_block, dest.to_block);
                 inserted_moves.push(pos, prio, *src_alloc, dest.alloc, self.vreg(dest.to_vreg));
             }

src/ion/process.rs

@@ -689,6 +689,7 @@ impl<'a, F: Function> Env<'a, F> {
                         index: empty_lr,
                     });
                     self.ctx.ranges[empty_lr].bundle = spill;
+                    self.ctx.ranges[empty_lr].vreg = vreg;
                     self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
                         range,
                         index: empty_lr,
@@ -754,6 +755,7 @@ impl<'a, F: Function> Env<'a, F> {
                         index: empty_lr,
                     });
                     self.ctx.ranges[empty_lr].bundle = spill;
+                    self.ctx.ranges[empty_lr].vreg = vreg;
                     self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
                         range,
                         index: empty_lr,