From fd5549793c81c3b9a64172403c213f60bfe0c8b4 Mon Sep 17 00:00:00 2001 From: Evan Carroll Date: Tue, 27 Oct 2026 13:51:58 -0500 Subject: [PATCH 1/2] fuzz: sometimes clobber nearly every register of a class at call-like instructions Make the fuzz generator emit instructions that clobber nearly every register of a class, the way a call does, so that spill bundles in the fuzzer-generated functions actually reach the loop-region second chance pass. One register per operand of the clobbered class stays free at the top of the file so that every preferred register is clobbered, keeping the instruction itself allocatable. --- src/fuzzing/func.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index 6bc8d067..96ec2fe1 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -560,13 +560,31 @@ impl Func { // is less than the number of registers in any single // class, so the resulting problem is always // allocatable. - for _ in 0..u.int_in_range(opts.num_clobbers_per_inst.clone())? { - let reg = u.int_in_range(0..=30)?; - let preg = PReg::new(reg, RegClass::arbitrary(u)?); - if operands.iter().any(has_fixed_def_with(preg)) { - continue; + if bool::arbitrary(u)? { + // Or clobber nearly every register of one class, as a + // call does, so that values live across this + // instruction must be spilled. One register per + // operand of that class stays free, at the top so + // that every preferred register is clobbered, which + // keeps the instruction itself allocatable. + let class = RegClass::arbitrary(u)?; + let free = operands.iter().filter(|op| op.class() == class).count(); + for reg in 0..32 - free { + let preg = PReg::new(reg, class); + if operands.iter().any(has_fixed_def_with(preg)) { + continue; + } + clobbers.push(preg); + } + } else { + for _ in 0..u.int_in_range(opts.num_clobbers_per_inst.clone())? { + let reg = u.int_in_range(0..=30)?; + let preg = PReg::new(reg, RegClass::arbitrary(u)?); + if operands.iter().any(has_fixed_def_with(preg)) { + continue; + } + clobbers.push(preg); } - clobbers.push(preg); } } } else if opts.clobbers && bool::arbitrary(u)? { From 71d1b6fb6f2cf0e773b6b99db9f1132f1fb58ef9 Mon Sep 17 00:00:00 2001 From: Evan Carroll Date: Tue, 27 Oct 2026 13:52:09 -0500 Subject: [PATCH 2/2] Give spilled bundles a register in the loops where they are used When a spill bundle that is live across a call fails the whole-range second chance, take the loops where its vregs are used, outermost first, and try to give each piece in the loop one register via a zero-cost probe. If the region contains a call, only non-preferred registers are probed. Pieces crossing a full clobber point or whose vreg has no use in the loop stay on the stack. After a successful probe, a vreg's pieces are kept only if its value is already in the spill slot on every way into the loop or it would be stored back at no more than one loop exit. Move insertion skips the store into a slot that already holds the value at loop exits and on block-parameter edges. --- src/ion/data_structures.rs | 27 ++ src/ion/moves.rs | 55 ++++ src/ion/process.rs | 2 + src/ion/spill.rs | 595 ++++++++++++++++++++++++++++++++++++- 4 files changed, 676 insertions(+), 3 deletions(-) diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index ba036a11..ff026856 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -130,6 +130,9 @@ pub struct LiveRange { #[repr(u32)] pub enum LiveRangeFlag { StartsAtDef = 1, + /// The value is also in its spillset's spill slot throughout the range, + /// so moving it there at the end of the range is redundant. + InSpillSlot = 2, } impl LiveRange { @@ -567,6 +570,21 @@ pub struct Ctx { pub(crate) scratch_removed_lrs_vregs: FxHashSet, pub(crate) scratch_workqueue_set: FxHashSet, + // For each block, the first and last block of the innermost loop + // containing it (see `try_allocating_regs_in_loops`). + pub(crate) scratch_loop_runs: Vec<(u32, u32)>, + // Per register class, the sorted points at which every allocatable + // register of that class is clobbered, and those at which every + // preferred one is (calls, when the preferred ones are caller-saved). + pub(crate) scratch_full_clobbers: [Vec; 3], + pub(crate) scratch_call_points: [Vec; 3], + // Candidate loop regions: (depth, first block, last block). + pub(crate) scratch_loop_regions: Vec<(u32, u32, u32)>, + // The uses of a spilled bundle's vregs, sorted by vreg and position. + pub(crate) scratch_loop_uses: Vec<(VRegIndex, ProgPoint)>, + // Whether any range has `LiveRangeFlag::InSpillSlot` in this function. + pub(crate) any_in_spill_slot: bool, + pub(crate) scratch_bump: UnsafeSendSync, } @@ -629,6 +647,12 @@ impl Default for Ctx { scratch_removed_lrs: Default::default(), scratch_removed_lrs_vregs: Default::default(), scratch_workqueue_set: Default::default(), + scratch_loop_runs: Default::default(), + scratch_full_clobbers: Default::default(), + scratch_call_points: Default::default(), + scratch_loop_regions: Default::default(), + scratch_loop_uses: Default::default(), + any_in_spill_slot: false, } } } @@ -973,6 +997,9 @@ pub struct Stats { pub spill_bundle_count: usize, pub spill_bundle_reg_probes: usize, pub spill_bundle_reg_success: usize, + pub spill_bundle_loop_region_probes: usize, + pub spill_bundle_loop_region_success: usize, + pub spill_slot_stores_elided: usize, pub blockparam_ins_count: usize, pub blockparam_outs_count: usize, pub halfmoves_count: usize, diff --git a/src/ion/moves.rs b/src/ion/moves.rs index ddabd9d8..c21521f4 100644 --- a/src/ion/moves.rs +++ b/src/ion/moves.rs @@ -73,6 +73,22 @@ impl<'a, F: Function> Env<'a, F> { } } + /// The spill slot that also holds `vreg` at the end of `block`, as marked + /// by the loop-region pass, if any. Only called when a value in a + /// register would be moved to the stack there. + fn spill_slot_holding(&self, vreg: VRegIndex, block: Block) -> Option { + if !self.ctx.any_in_spill_slot { + return None; + } + let exit = self.ctx.cfginfo.block_exit[block.index()]; + let entry = self.ctx.vregs[vreg].ranges.iter().find(|entry| { + entry.range.contains_point(exit) + && self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::InSpillSlot) + })?; + let spillset = self.ctx.bundles[self.ctx.ranges[entry.index].bundle].spillset; + Some(self.ctx.spillslots[self.ctx.spillsets[spillset].slot.index()].alloc) + } + pub fn apply_allocations_and_insert_moves(&mut self) -> InsertedMoves { trace!("apply_allocations_and_insert_moves"); trace!("blockparam_ins: {:?}", self.blockparam_ins); @@ -631,6 +647,21 @@ impl<'a, F: Function> Env<'a, F> { trace!("processing inter-block moves for {}", vreg); for dest in inter_block_dests.drain(..) { let src = inter_block_sources[&dest.from]; + if src.is_reg() + && dest.alloc.is_stack() + && self.spill_slot_holding(VRegIndex::new(vreg.vreg()), dest.from) + == Some(dest.alloc) + { + trace!( + " -> {} already holds {} between {:?} and {:?}", + dest.alloc, + vreg, + dest.from, + dest.to + ); + self.output.stats.spill_slot_stores_elided += 1; + continue; + } trace!( " -> moving from {} to {} between {:?} and {:?}", @@ -653,9 +684,33 @@ impl<'a, F: Function> Env<'a, F> { self.output.stats.halfmoves_count += block_param_dests.len(); trace!("processing block-param moves"); + let mut block_param_args = FxHashMap::::default(); for dest in block_param_dests { let src = dest.source(); let src_alloc = block_param_sources.get(&src).unwrap(); + if self.ctx.any_in_spill_slot && src_alloc.is_reg() && dest.alloc.is_stack() { + // Which value is passed, for the few moves that could + // store one already in its slot. + if block_param_args.is_empty() { + block_param_args.extend(self.blockparam_outs.iter().map(|out| { + ( + BlockparamSourceKey::new(out.from_block, out.to_vreg), + out.from_vreg, + ) + })); + } + let from_vreg = block_param_args[&src]; + if self.spill_slot_holding(from_vreg, dest.from_block) == Some(dest.alloc) { + trace!( + " -> {} already holds the value of v{} from block{}", + dest.alloc, + dest.to_vreg.index(), + dest.from_block.index() + ); + self.output.stats.spill_slot_stores_elided += 1; + continue; + } + } let (pos, prio) = choose_move_location(self, dest.from_block, dest.to_block); inserted_moves.push(pos, prio, *src_alloc, dest.alloc, self.vreg(dest.to_vreg)); } diff --git a/src/ion/process.rs b/src/ion/process.rs index 57b81e69..1464eb81 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -689,6 +689,7 @@ impl<'a, F: Function> Env<'a, F> { index: empty_lr, }); self.ctx.ranges[empty_lr].bundle = spill; + self.ctx.ranges[empty_lr].vreg = vreg; self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { range, index: empty_lr, @@ -754,6 +755,7 @@ impl<'a, F: Function> Env<'a, F> { index: empty_lr, }); self.ctx.ranges[empty_lr].bundle = spill; + self.ctx.ranges[empty_lr].vreg = vreg; self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { range, index: empty_lr, diff --git a/src/ion/spill.rs b/src/ion/spill.rs index 8eb44cb9..42430742 100644 --- a/src/ion/spill.rs +++ b/src/ion/spill.rs @@ -13,15 +13,28 @@ //! Spillslot allocation. use super::{ - AllocRegResult, Env, LiveRangeKey, PRegIndex, RegTraversalIter, SpillSetIndex, SpillSlotData, - SpillSlotIndex, + AllocRegResult, CodeRange, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, + LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, SpillSetIndex, + SpillSlotData, SpillSlotIndex, UseList, VRegIndex, }; -use crate::{Allocation, Function, SpillSlot}; +use crate::{Allocation, Block, Function, PRegSet, ProgPoint, RegClass, SpillSlot}; +use smallvec::SmallVec; + +/// Which of the per-function data used by `try_allocating_regs_in_loops` +/// has been computed. +#[derive(Default)] +struct LoopInfoComputed { + loop_runs: bool, + clobber_points: [bool; 3], +} impl<'a, F: Function> Env<'a, F> { pub fn try_allocating_regs_for_spilled_bundles(&mut self) { trace!("allocating regs for spilled bundles"); let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts); + let mut failed = core::mem::take(&mut self.ctx.scratch_bundle); + failed.clear(); + self.ctx.any_in_spill_slot = false; for i in 0..self.ctx.spilled_bundles.len() { let bundle = self.ctx.spilled_bundles[i]; // don't borrow self @@ -62,11 +75,549 @@ impl<'a, F: Function> Env<'a, F> { self.ctx.bundles[bundle].spillset ); self.ctx.spillsets[self.ctx.bundles[bundle].spillset].required = true; + failed.push(bundle); + } + } + + // Only once every spilled bundle has had its chance at a register + // as a whole do we carve loops out of the ones that failed, so that + // a loop region cannot take the register that a later bundle would + // have fit in entirely. + if !failed.is_empty() && self.ctx.cfginfo.approx_loop_depth.iter().any(|&d| d > 0) { + let mut computed = LoopInfoComputed::default(); + for &bundle in &failed { + self.try_allocating_regs_in_loops(bundle, &mut computed, &mut scratch); } } + self.ctx.scratch_bundle = failed; self.ctx.scratch_conflicts = scratch; } + /// Second chance for a spilled bundle that did not fit in a register + /// as a whole, usually because it is live across a call that clobbers + /// every register of its class: give a register to its pieces inside + /// loops that contain uses, so that values used in a loop are not + /// loaded from and stored to the stack on every iteration. + /// + /// Loops are tried outermost first, and a loop's pieces are only taken + /// out of the spill bundle if they all fit in one register, so the + /// moves this adds are at loop entries and exits. Pieces that are live + /// across a point clobbering every register of the class, or whose vreg is + /// neither used nor defined in the loop, are left on the stack. + fn try_allocating_regs_in_loops( + &mut self, + bundle: LiveBundleIndex, + computed: &mut LoopInfoComputed, + conflicts: &mut LiveBundleVec, + ) { + let spillset = self.ctx.bundles[bundle].spillset; + let class = self.ctx.spillsets[spillset].class; + let hint = self.ctx.spillsets[spillset].hint.as_valid(); + let limit = self.ctx.bundles[bundle].limit.map(|l| l as usize); + + // Only bundles live across a call are of interest here; the others + // failed for lack of registers, and their loops seldom have any free. + if !computed.clobber_points[class as usize] { + self.compute_clobber_points(class); + computed.clobber_points[class as usize] = true; + } + let call_points = &self.ctx.scratch_call_points[class as usize]; + if !self.ctx.bundles[bundle] + .ranges + .iter() + .any(|entry| any_point_in(call_points, entry.range)) + { + return; + } + + // Gather every use of the bundle's vregs once, sorted, so that the + // searches below stay logarithmic however many ranges they have. + let mut uses = core::mem::take(&mut self.ctx.scratch_loop_uses); + uses.clear(); + let mut vregs: SmallVec<[VRegIndex; 4]> = self.ctx.bundles[bundle] + .ranges + .iter() + .map(|entry| self.ctx.ranges[entry.index].vreg) + .collect(); + vregs.sort_unstable(); + vregs.dedup(); + for &vreg in &vregs { + for vreg_entry in &self.ctx.vregs[vreg].ranges { + for u in &self.ctx.ranges[vreg_entry.index].uses { + uses.push((vreg, u.pos)); + } + } + } + uses.sort_unstable(); + + // Find the blocks in loops where the bundle's vregs are used, within + // the bundle's ranges. Consecutive uses are mostly in one block. + let mut use_blocks: SmallVec<[u32; 16]> = SmallVec::new(); + let ranges = &self.ctx.bundles[bundle].ranges; + for &(vreg, pos) in &uses { + let i = ranges.partition_point(|entry| entry.range.to <= pos); + if i == ranges.len() + || !ranges[i].range.contains_point(pos) + || self.ctx.ranges[ranges[i].index].vreg != vreg + { + continue; + } + let block = self.ctx.cfginfo.insn_block[pos.inst().index()].raw_u32(); + if self.ctx.cfginfo.approx_loop_depth[block as usize] > 0 + && use_blocks.last() != Some(&block) + { + use_blocks.push(block); + } + } + if use_blocks.is_empty() { + self.ctx.scratch_loop_uses = uses; + return; + } + if !computed.loop_runs { + self.compute_loop_runs(); + computed.loop_runs = true; + } + + // Collect the loops, at every depth, around each of those blocks. + let mut regions = core::mem::take(&mut self.ctx.scratch_loop_regions); + regions.clear(); + let depth = &self.ctx.cfginfo.approx_loop_depth; + let runs = &mut self.ctx.scratch_loop_runs; + let mut last_innermost = None; + for &block in &use_blocks { + let (mut start, mut end) = loop_run(depth, runs, block); + if last_innermost == Some((start, end)) { + continue; + } + last_innermost = Some((start, end)); + let mut d = depth[block as usize]; + loop { + regions.push((d, start, end)); + let left = if start > 0 { + depth[start as usize - 1] + } else { + 0 + }; + let right = depth.get(end as usize + 1).copied().unwrap_or(0); + let parent = core::cmp::max(left, right); + if parent == 0 { + break; + } + let outer = if left >= right { start - 1 } else { end + 1 }; + (start, end) = loop_run(depth, runs, outer); + d = parent; + } + } + regions.sort_unstable(); + regions.dedup(); + + let mut probe = LiveBundleIndex::invalid(); + let mut loop_pieces = SmallVec::new(); + let mut pieces: SmallVec<[(LiveRangeIndex, CodeRange); 4]> = SmallVec::new(); + for &(_, start, end) in ®ions { + let region = CodeRange { + from: self.ctx.cfginfo.block_entry[start as usize], + to: self.ctx.cfginfo.block_exit[end as usize].next(), + }; + + pieces.clear(); + let ranges = &self.ctx.bundles[bundle].ranges; + let first = ranges.partition_point(|entry| entry.range.to <= region.from); + let mut whole = first == 0; + let mut past = first; + for entry in &ranges[first..] { + if entry.range.from >= region.to { + break; + } + past += 1; + let piece = CodeRange { + from: core::cmp::max(entry.range.from, region.from), + to: core::cmp::min(entry.range.to, region.to), + }; + if piece != entry.range { + whole = false; + } + // A value defined in the loop has no use in its piece, which + // starts after the def, so look for uses in the whole loop. + if !any_use_in(&uses, self.ctx.ranges[entry.index].vreg, region) + || any_point_in(&self.ctx.scratch_full_clobbers[class as usize], piece) + { + whole = false; + continue; + } + pieces.push((entry.index, piece)); + } + // Nothing to gain, or the whole bundle, which already failed. + if pieces.is_empty() || (whole && past == ranges.len()) { + continue; + } + + if !probe.is_valid() { + probe = self.ctx.bundles.add(self.ctx.scratch_bump.clone()); + self.ctx.bundles[probe].spillset = spillset; + } + self.ctx.bundles[probe].ranges.clear(); + for &(_, piece) in &pieces { + // The real ranges are created only if the probe succeeds; + // their indices are patched into the preg's map then. + self.ctx.bundles[probe].ranges.push(LiveRangeListEntry { + range: piece, + index: LiveRangeIndex::invalid(), + }); + } + + self.ctx.output.stats.spill_bundle_loop_region_probes += 1; + // Every preferred register is clobbered at a call, so a region + // with one can only go in a non-preferred register. + let call_points = &self.ctx.scratch_call_points[class as usize]; + let skip = if pieces + .iter() + .any(|&(_, piece)| any_point_in(call_points, piece)) + { + self.env.preferred_regs_by_class[class as usize] + } else { + PRegSet::empty() + }; + let mut allocated = None; + for preg in RegTraversalIter::new(self.env, class, None, hint, bundle.index(), limit) { + if skip.contains(preg) { + continue; + } + let preg_idx = PRegIndex::new(preg.index()); + if let AllocRegResult::Allocated(_) = + self.try_to_allocate_bundle_to_reg(probe, preg_idx, Some(0), conflicts) + { + allocated = Some(preg_idx); + break; + } + } + let Some(preg_idx) = allocated else { + continue; + }; + // A value not already in the spill slot on the way into the loop, + // as one defined in it, must be stored at each exit where it is + // live instead of once at its def. Keep it on the stack if that is + // more than one exit, so as not to add stores. This is judged only + // for the few regions that fit, and what remains still fits. + let mut verdicts: SmallVec<[(VRegIndex, bool); 4]> = SmallVec::new(); + for &(lr, _) in &pieces { + let vreg = self.ctx.ranges[lr].vreg; + if !verdicts.iter().any(|v| v.0 == vreg) { + verdicts.push((vreg, true)); + } + } + for i in 0..verdicts.len() { + let vreg = verdicts[i].0; + let live_in = pieces.iter().any(|&(lr, piece)| { + piece.from == region.from && self.ctx.ranges[lr].vreg == vreg + }); + verdicts[i].1 = (live_in + && self.in_spill_bundle_on_entry(bundle, vreg, start, end)) + || self.loop_exits_holding(bundle, vreg, start, end, &verdicts) <= 1; + } + if verdicts.iter().any(|v| !v.1) { + let preg_allocs = &mut self.ctx.pregs[preg_idx.index()].allocations.btree; + let ranges = &self.ctx.ranges; + pieces.retain(|&mut (lr, piece)| { + let vreg = ranges[lr].vreg; + let keep = verdicts.iter().find(|v| v.0 == vreg).unwrap().1; + if !keep { + preg_allocs.remove(&LiveRangeKey::from_range(&piece)); + } + keep + }); + self.ctx.bundles[probe].ranges.clear(); + if pieces.is_empty() { + self.ctx.bundles[probe].allocation = Allocation::none(); + continue; + } + for &(_, piece) in &pieces { + self.ctx.bundles[probe].ranges.push(LiveRangeListEntry { + range: piece, + index: LiveRangeIndex::invalid(), + }); + } + } + trace!( + "loop region {:?} of spilled bundle {:?} allocated to {:?} as bundle {:?}", + region, + bundle, + preg_idx, + probe + ); + self.ctx.output.stats.spill_bundle_loop_region_success += 1; + self.split_out_loop_pieces( + bundle, + probe, + preg_idx, + &pieces, + (start, end), + &mut loop_pieces, + ); + probe = LiveBundleIndex::invalid(); + } + + // A value live into a loop and in the spill slot on every way into + // it stays there, as the slot is reserved for the whole spillset: none + // of its pieces in the loop needs a store at the exits. Judge this once + // all the loops have been split out, as a later one may take the value + // out of the slot on a way into an earlier one. + let mut in_slot: SmallVec<[(VRegIndex, u32, u32); 4]> = SmallVec::new(); + for &(lr, start, end, live_in) in &loop_pieces { + let vreg = self.ctx.ranges[lr].vreg; + if live_in && self.in_spill_bundle_on_entry(bundle, vreg, start, end) { + in_slot.push((vreg, start, end)); + } + } + if !in_slot.is_empty() { + for &(lr, start, end, _) in &loop_pieces { + if in_slot.contains(&(self.ctx.ranges[lr].vreg, start, end)) { + self.ctx.ranges[lr].set_flag(LiveRangeFlag::InSpillSlot); + self.ctx.any_in_spill_slot = true; + } + } + } + + self.ctx.scratch_loop_regions = regions; + self.ctx.scratch_loop_uses = uses; + } + + /// Moves `pieces` out of the ranges of spilled `bundle` into `region_bundle`, + /// which has just been allocated to `preg`: each range covering a piece is cut + /// into the part before it, which keeps its index, the piece, and the part + /// after it, both new ranges. + fn split_out_loop_pieces( + &mut self, + bundle: LiveBundleIndex, + region_bundle: LiveBundleIndex, + preg: PRegIndex, + pieces: &[(LiveRangeIndex, CodeRange)], + (start, end): (u32, u32), + loop_pieces: &mut SmallVec<[(LiveRangeIndex, u32, u32, bool); 4]>, + ) { + let bump = self.ctx.bump(); + let old_ranges = core::mem::replace( + &mut self.ctx.bundles[bundle].ranges, + LiveRangeList::new_in(bump.clone()), + ); + let mut next_piece = 0; + for entry in old_ranges { + let lr = entry.index; + if next_piece == pieces.len() || pieces[next_piece].0 != lr { + self.ctx.bundles[bundle].ranges.push(entry); + continue; + } + let piece = pieces[next_piece].1; + let vreg = self.ctx.ranges[lr].vreg; + + let piece_lr = self.ctx.ranges.add(piece, bump.clone()); + self.ctx.ranges[piece_lr].vreg = vreg; + self.ctx.ranges[piece_lr].bundle = region_bundle; + // Whether a value live into the loop is in the spill slot there is + // judged later, from the loop's predecessors. + let live_in = piece.from == self.ctx.cfginfo.block_entry[start as usize]; + loop_pieces.push((piece_lr, start, end, live_in)); + self.ctx.bundles[region_bundle].ranges[next_piece].index = piece_lr; + *self.ctx.pregs[preg.index()] + .allocations + .btree + .get_mut(&LiveRangeKey::from_range(&piece)) + .unwrap() = piece_lr; + self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { + range: piece, + index: piece_lr, + }); + next_piece += 1; + + let tail_lr = if piece.to < entry.range.to { + let tail = CodeRange { + from: piece.to, + to: entry.range.to, + }; + let tail_lr = self.ctx.ranges.add(tail, bump.clone()); + self.ctx.ranges[tail_lr].vreg = vreg; + self.ctx.ranges[tail_lr].bundle = bundle; + self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { + range: tail, + index: tail_lr, + }); + Some(tail_lr) + } else { + None + }; + + let uses = + core::mem::replace(&mut self.ctx.ranges[lr].uses, UseList::new_in(bump.clone())); + for u in uses { + let to = if u.pos < piece.from { + lr + } else if u.pos < piece.to { + piece_lr + } else { + tail_lr.unwrap() + }; + self.ctx.ranges[to].uses.push(u); + } + + if piece.from > entry.range.from { + self.ctx.ranges[lr].range.to = piece.from; + self.ctx.bundles[bundle].ranges.push(LiveRangeListEntry { + range: self.ctx.ranges[lr].range, + index: lr, + }); + } else { + // The piece starts the range, so it takes over its flags and + // the old range goes away. + let flags = self.ctx.ranges[lr].flag_word(); + self.ctx.ranges[piece_lr].merge_flags(flags); + self.ctx.vregs[vreg].ranges.retain(|e| e.index != lr); + } + if let Some(tail_lr) = tail_lr { + self.ctx.bundles[bundle].ranges.push(LiveRangeListEntry { + range: self.ctx.ranges[tail_lr].range, + index: tail_lr, + }); + } + } + debug_assert_eq!(next_piece, pieces.len()); + } + + /// Whether `vreg` is in a range of spilled `bundle` at the end of each + /// predecessor, outside the run of blocks `start..=end`, of a block in it, + /// the first block having at least one such predecessor. + fn in_spill_bundle_on_entry( + &self, + bundle: LiveBundleIndex, + vreg: VRegIndex, + start: u32, + end: u32, + ) -> bool { + let ranges = &self.ctx.bundles[bundle].ranges; + // The loop's first block must have a way in, or the value may be + // defined there, as at the entry block, and not yet be stored. + let first = Block::new(start as usize); + if self + .func + .block_preds(first) + .iter() + .all(|pred| pred.raw_u32() >= start && pred.raw_u32() <= end) + { + return false; + } + (start..=end).all(|block| { + self.func + .block_preds(Block::new(block as usize)) + .iter() + .filter(|pred| pred.raw_u32() < start || pred.raw_u32() > end) + .all(|pred| { + let exit = self.ctx.cfginfo.block_exit[pred.index()]; + let i = ranges.partition_point(|entry| entry.range.to <= exit); + i < ranges.len() + && ranges[i].range.contains_point(exit) + && self.ctx.ranges[ranges[i].index].vreg == vreg + }) + }) + } + + /// The number, up to 2, of edges on which `vreg`, in a range of `bundle` + /// at the end of a block in the run `start..=end`, is moved out of the + /// loop's registers: those leaving the run, and those in it passing the + /// value to a block parameter none of whose pieces, `in_loop`, is in them. + fn loop_exits_holding( + &self, + bundle: LiveBundleIndex, + vreg: VRegIndex, + start: u32, + end: u32, + in_loop: &[(VRegIndex, bool)], + ) -> usize { + let ranges = &self.ctx.bundles[bundle].ranges; + let mut exits = 0; + for block in start..=end { + let exit = self.ctx.cfginfo.block_exit[block as usize]; + let i = ranges.partition_point(|entry| entry.range.to <= exit); + if i == ranges.len() + || !ranges[i].range.contains_point(exit) + || self.ctx.ranges[ranges[i].index].vreg != vreg + { + continue; + } + for succ in self.func.block_succs(Block::new(block as usize)) { + if succ.raw_u32() < start || succ.raw_u32() > end { + exits += 1; + if exits > 1 { + return exits; + } + } + } + } + let outs = &self.ctx.blockparam_outs; + let first = outs.partition_point(|out| out.from_vreg < vreg); + for out in &outs[first..] { + if out.from_vreg != vreg { + break; + } + let (from, to) = (out.from_block.raw_u32(), out.to_block.raw_u32()); + if from < start + || from > end + || to < start + || to > end + || in_loop.iter().any(|v| v.0 == out.to_vreg) + { + continue; + } + exits += 1; + if exits > 1 { + break; + } + } + exits + } + + /// Prepares the cache of `loop_run`. + fn compute_loop_runs(&mut self) { + let num_blocks = self.ctx.cfginfo.approx_loop_depth.len(); + let runs = &mut self.ctx.scratch_loop_runs; + runs.clear(); + runs.resize(num_blocks, (u32::MAX, u32::MAX)); + } + + /// Collects the points at which every allocatable register of `class` is + /// clobbered, and those at which every preferred register is. Either is a + /// fixed reservation on the first preferred register, so it is enough to + /// look at the fixed reservations of that one. + fn compute_clobber_points(&mut self, class: RegClass) { + let class = class as usize; + let mut preferred = self.env.preferred_regs_by_class[class]; + let allocatable = preferred | self.env.non_preferred_regs_by_class[class]; + if preferred == PRegSet::empty() { + preferred = allocatable; + } + let full = &mut self.ctx.scratch_full_clobbers[class]; + let calls = &mut self.ctx.scratch_call_points[class]; + full.clear(); + calls.clear(); + let Some(reg) = preferred.into_iter().chain(allocatable).next() else { + return; + }; + for (key, lr) in &self.ctx.pregs[reg.index()].allocations.btree { + if lr.is_valid() { + continue; + } + let point = ProgPoint::after(ProgPoint::from_index(key.from).inst()); + if calls.last() == Some(&point) || full.last() == Some(&point) { + continue; + } + let clobbers = self.func.inst_clobbers(point.inst()); + if clobbers & preferred == preferred { + calls.push(point); + } + if clobbers & allocatable == allocatable { + full.push(point); + } + } + } + pub fn spillslot_can_fit_spillset( &mut self, spillslot: SpillSlotIndex, @@ -171,3 +722,41 @@ impl<'a, F: Function> Env<'a, F> { Allocation::stack(SpillSlot::new(slot as usize)) } } + +/// Whether one of the sorted `points` lies in `range`. +fn any_point_in(points: &[ProgPoint], range: CodeRange) -> bool { + let i = points.partition_point(|&p| p < range.from); + i < points.len() && points[i] < range.to +} + +/// Whether `vreg` has a use in `range`, given all uses sorted by vreg and +/// position. +fn any_use_in(uses: &[(VRegIndex, ProgPoint)], vreg: VRegIndex, range: CodeRange) -> bool { + let i = uses.partition_point(|&u| u < (vreg, range.from)); + i < uses.len() && uses[i].0 == vreg && uses[i].1 < range.to +} + +/// The first and last block of the innermost loop containing `block`, that +/// is the maximal run of consecutive blocks around it at least as deep. +/// Cached in `runs` for every block of that depth in the run, which is +/// their innermost loop too. +fn loop_run(depth: &[u32], runs: &mut [(u32, u32)], block: u32) -> (u32, u32) { + let b = block as usize; + if runs[b].0 != u32::MAX { + return runs[b]; + } + let d = depth[b]; + let start = depth[..b].iter().rposition(|&x| x < d).map_or(0, |i| i + 1); + let end = depth[b..] + .iter() + .position(|&x| x < d) + .map_or(depth.len(), |i| b + i) + - 1; + let run = (start as u32, end as u32); + for i in start..=end { + if depth[i] == d { + runs[i] = run; + } + } + run +}