From 5a5fab3bd8e0409a447e75b1332f98b4c9520018 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 12 Nov 2026 16:09:06 -0800 Subject: [PATCH 1/4] Add `Function::is_block_cold` hint --- src/cfg.rs | 4 ++++ src/fuzzing/func.rs | 20 ++++++++++++++++++-- src/fuzzing/ion.rs | 1 + src/lib.rs | 9 +++++++++ src/serialize.rs | 19 ++++++++++++++++++- 5 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/cfg.rs b/src/cfg.rs index 499ab835..93151eaa 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -34,6 +34,8 @@ pub struct CFGInfo { /// indices. Otherwise, it will be approximate, but should still /// be usable for heuristic purposes. pub approx_loop_depth: Vec, + /// Whether each block is cold. + pub cold: Vec, } impl CFGInfo { @@ -63,6 +65,7 @@ impl CFGInfo { .block_exit .repopulate(nb, ProgPoint::before(Inst::invalid())); let (backedge_in, backedge_out) = ctx.backedge.repopulate(nb * 2, 0).split_at_mut(nb); + let cold = self.cold.repopulate(nb, false); for block in 0..f.num_blocks() { let block = Block::new(block); @@ -71,6 +74,7 @@ impl CFGInfo { } block_entry[block.index()] = ProgPoint::before(f.block_insns(block).first()); block_exit[block.index()] = ProgPoint::after(f.block_insns(block).last()); + cold[block.index()] = f.is_block_cold(block); // Check critical edge condition: if there is more than // one predecessor, each must have only one successor diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index 6bc8d067..cd7f870d 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -57,6 +57,7 @@ pub struct Func { num_vregs: usize, reftype_vregs: Vec, debug_value_labels: Vec<(VReg, Inst, Inst, u32)>, + cold_blocks: Vec, } impl Function for Func { @@ -129,6 +130,10 @@ impl Function for Func { RegClass::Vector => 2, } } + + fn is_block_cold(&self, block: Block) -> bool { + self.cold_blocks[block.index()] + } } struct FuncBuilder { @@ -153,6 +158,7 @@ impl FuncBuilder { num_vregs: 0, reftype_vregs: vec![], debug_value_labels: vec![], + cold_blocks: vec![], }, insts_per_block: vec![], } @@ -167,6 +173,7 @@ impl FuncBuilder { self.f.block_succs.push(vec![]); self.f.block_params_in.push(vec![]); self.f.block_params_out.push(vec![]); + self.f.cold_blocks.push(false); self.insts_per_block.push(vec![]); b } @@ -370,6 +377,7 @@ pub struct Options { pub fixed_def_clobbers: bool, pub reftypes: bool, pub callsite_ish_constraints: bool, + pub cold_blocks: bool, pub num_blocks: RangeInclusive, pub num_vregs_per_block: RangeInclusive, pub num_uses_per_inst: RangeInclusive, @@ -387,6 +395,7 @@ impl Options { fixed_def_clobbers: false, reftypes: false, callsite_ish_constraints: false, + cold_blocks: false, num_blocks: 1..=100, num_vregs_per_block: 5..=15, num_uses_per_inst: 0..=10, @@ -449,6 +458,12 @@ impl Func { } builder.compute_doms(); + if opts.cold_blocks { + for block in 0..num_blocks { + builder.f.cold_blocks[block] = u.ratio(1, 4)?; + } + } + let alloc_vreg = |builder: &mut FuncBuilder, u: &mut Unstructured| { let vreg = VReg::new(builder.f.num_vregs, RegClass::arbitrary(u)?); builder.f.num_vregs += 1; @@ -703,10 +718,11 @@ impl core::fmt::Debug for Func { }) .collect::>() .join(", "); + let cold = if self.cold_blocks[i] { " cold" } else { "" }; write!( f, - " block{}({}): # succs:{:?} preds:{:?}\n", - i, params_in, succs, preds + " block{}({}){}: # succs:{:?} preds:{:?}\n", + i, params_in, cold, succs, preds )?; for inst in blockrange.iter() { write!( diff --git a/src/fuzzing/ion.rs b/src/fuzzing/ion.rs index c76037cd..5228b14a 100644 --- a/src/fuzzing/ion.rs +++ b/src/fuzzing/ion.rs @@ -14,6 +14,7 @@ const OPTIONS: func::Options = func::Options { fixed_def_clobbers: true, reftypes: true, callsite_ish_constraints: true, + cold_blocks: true, ..func::Options::DEFAULT }; diff --git a/src/lib.rs b/src/lib.rs index 5eab7d38..15cf8073 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1352,6 +1352,15 @@ pub trait Function { fn allow_multiple_vreg_defs(&self) -> bool { false } + + /// Is the given block cold, i.e. rarely executed? + /// + /// The allocator may use this as a hint to keep spills, reloads, and + /// other moves out of hot code and in cold blocks instead. It does not + /// affect correctness. + fn is_block_cold(&self, _block: Block) -> bool { + false + } } /// A position before or after an instruction at which we can make an diff --git a/src/serialize.rs b/src/serialize.rs index e98d8831..8ee46865 100644 --- a/src/serialize.rs +++ b/src/serialize.rs @@ -50,6 +50,8 @@ pub struct SerializableFunction { spillslot_size: Vec, multi_spillslot_named_by_last_slot: bool, allow_multiple_vreg_defs: bool, + #[serde(default)] + cold_blocks: Vec, } impl SerializableFunction { @@ -119,6 +121,9 @@ impl SerializableFunction { .to_vec(), multi_spillslot_named_by_last_slot: func.multi_spillslot_named_by_last_slot(), allow_multiple_vreg_defs: func.allow_multiple_vreg_defs(), + cold_blocks: (0..func.num_blocks()) + .map(|i| func.is_block_cold(Block::new(i))) + .collect(), } } @@ -196,6 +201,13 @@ impl Function for SerializableFunction { fn allow_multiple_vreg_defs(&self) -> bool { self.allow_multiple_vreg_defs } + + fn is_block_cold(&self, block: Block) -> bool { + self.cold_blocks + .get(block.index()) + .copied() + .unwrap_or(false) + } } impl fmt::Debug for SerializableFunction { @@ -255,9 +267,14 @@ impl fmt::Debug for SerializableFunction { }) .collect::>() .join(", "); + let cold = if self.is_block_cold(Block::new(i)) { + " cold" + } else { + "" + }; write!( f, - " block{i}({params_in}): # succs:{succs:?} preds:{preds:?}\n", + " block{i}({params_in}){cold}: # succs:{succs:?} preds:{preds:?}\n", )?; for inst in blockrange.iter() { let ops: Vec<_> = self From aca9f324a2aa91964cac86e6d9f514f0feaebac9 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 12 Nov 2026 16:14:48 -0800 Subject: [PATCH 2/4] ion: give cold blocks zero loop depth for spill weight --- src/cfg.rs | 12 ++++++++++++ src/ion/data_structures.rs | 1 + src/ion/liveranges.rs | 2 +- src/ion/process.rs | 12 ++++++------ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/cfg.rs b/src/cfg.rs index 93151eaa..6cd56141 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -140,4 +140,16 @@ impl CFGInfo { Ok(()) } + + /// The loop depth to use when weighing spill costs in `block`. + /// + /// Cold blocks are weighted as if they were outside of every loop, so that + /// their uses don't make hot code pay to keep values in registers. + pub fn spill_weight_loop_depth(&self, block: Block) -> u32 { + if self.cold[block.index()] { + 0 + } else { + self.approx_loop_depth[block.index()] + } + } } diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index ba036a11..d8adbee4 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -968,6 +968,7 @@ pub struct Stats { pub splits_conflicts: usize, pub splits_defs: usize, pub splits_all: usize, + pub splits_cold: usize, pub final_liverange_count: usize, pub final_bundle_count: usize, pub spill_bundle_count: usize, diff --git a/src/ion/liveranges.rs b/src/ion/liveranges.rs index 09040f6c..3de12be7 100644 --- a/src/ion/liveranges.rs +++ b/src/ion/liveranges.rs @@ -216,7 +216,7 @@ impl<'a, F: Function> Env<'a, F> { let operand = u.operand; let constraint = operand.constraint(); let block = self.cfginfo.insn_block[u.pos.inst().index()]; - let loop_depth = self.cfginfo.approx_loop_depth[block.index()] as usize; + let loop_depth = self.cfginfo.spill_weight_loop_depth(block) as usize; let weight = spill_weight_from_constraint( constraint, loop_depth, diff --git a/src/ion/process.rs b/src/ion/process.rs index 57b81e69..dba5c75c 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -1131,10 +1131,9 @@ impl<'a, F: Function> Env<'a, F> { lowest_cost_evict_conflict_set.extend(bundles); } - let loop_depth = - self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block - [first_conflict_point.inst().index()] - .index()]; + let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth( + self.ctx.cfginfo.insn_block[first_conflict_point.inst().index()], + ); let move_cost = spill_weight_from_constraint( OperandConstraint::Reg, loop_depth as usize, @@ -1153,8 +1152,9 @@ impl<'a, F: Function> Env<'a, F> { AllocRegResult::ConflictWithFixed(max_cost, point) => { trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); - let loop_depth = self.ctx.cfginfo.approx_loop_depth - [self.ctx.cfginfo.insn_block[point.inst().index()].index()]; + let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth( + self.ctx.cfginfo.insn_block[point.inst().index()], + ); let move_cost = spill_weight_from_constraint( OperandConstraint::Reg, loop_depth as usize, From 9dd6221230ebcb725cb7441ed3c6a197215a4bb4 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 12 Nov 2026 16:20:13 -0800 Subject: [PATCH 3/4] ion: split the cold portion out of bundles on fixed conflicts in cold code We do not count this towards the bundle's attempts since a bundle can only be split into hot and cold portions once, and each new sub-bundle is homogenous forever after. --- src/cfg.rs | 9 +- src/ion/process.rs | 237 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 198 insertions(+), 48 deletions(-) diff --git a/src/cfg.rs b/src/cfg.rs index 6cd56141..6cd90627 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -36,6 +36,8 @@ pub struct CFGInfo { pub approx_loop_depth: Vec, /// Whether each block is cold. pub cold: Vec, + /// Is any block cold? + pub any_cold: bool, } impl CFGInfo { @@ -66,6 +68,7 @@ impl CFGInfo { .repopulate(nb, ProgPoint::before(Inst::invalid())); let (backedge_in, backedge_out) = ctx.backedge.repopulate(nb * 2, 0).split_at_mut(nb); let cold = self.cold.repopulate(nb, false); + let mut any_cold = false; for block in 0..f.num_blocks() { let block = Block::new(block); @@ -74,7 +77,9 @@ impl CFGInfo { } block_entry[block.index()] = ProgPoint::before(f.block_insns(block).first()); block_exit[block.index()] = ProgPoint::after(f.block_insns(block).last()); - cold[block.index()] = f.is_block_cold(block); + let block_is_cold = f.is_block_cold(block); + cold[block.index()] = block_is_cold; + any_cold |= block_is_cold; // Check critical edge condition: if there is more than // one predecessor, each must have only one successor @@ -117,6 +122,8 @@ impl CFGInfo { } } + self.any_cold = any_cold; + let approx_loop_depth = self.approx_loop_depth.cleared(); let mut backedge_stack: SmallVec<[u32; 4]> = smallvec![]; let mut cur_depth = 0; diff --git a/src/ion/process.rs b/src/ion/process.rs index dba5c75c..b6de1c9f 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -13,21 +13,21 @@ //! Main allocation loop that processes bundles. use super::{ - spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, - LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, - Requirement, SpillWeight, UseList, VRegIndex, + Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, LiveRangeKey, + LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, Requirement, SpillWeight, + UseList, VRegIndex, spill_weight_from_constraint, }; use crate::{ + Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, + RegAllocError, ion::data_structures::{ - CodeRange, Use, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET, + BUNDLE_MAX_NORMAL_SPILL_WEIGHT, CodeRange, MAX_SPLITS_PER_SPILLSET, MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, - MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT, + MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT, Use, }, - Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, - RegAllocError, }; use core::fmt::Debug; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum AllocRegResult<'a> { @@ -214,8 +214,7 @@ impl<'a, F: Function> Env<'a, F> { pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { trace!( "evicting bundle {:?}: alloc {:?}", - bundle, - self.ctx.bundles[bundle].allocation + bundle, self.ctx.bundles[bundle].allocation ); let preg = match self.ctx.bundles[bundle].allocation.as_reg() { Some(preg) => preg, @@ -348,8 +347,7 @@ impl<'a, F: Function> Env<'a, F> { let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio; trace!( " -> dividing by prio {}; final weight {}", - self.ctx.bundles[bundle].prio, - final_weight + self.ctx.bundles[bundle].prio, final_weight ); core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight) } else { @@ -449,8 +447,7 @@ impl<'a, F: Function> Env<'a, F> { trace!( "bundle_start = {:?} bundle_end = {:?}", - bundle_start, - bundle_end + bundle_start, bundle_end ); // Is the bundle at most spanning one instruction? Then @@ -660,9 +657,7 @@ impl<'a, F: Function> Env<'a, F> { .unwrap(); trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - bundle, - entry.index, - spill + bundle, entry.index, spill ); self.ctx.bundles[spill].ranges.push(entry); self.ctx.bundles[bundle].ranges.pop(); @@ -695,15 +690,12 @@ impl<'a, F: Function> Env<'a, F> { }); trace!( " -> bundle {:?} range {:?}: last use implies split point {:?}", - bundle, - entry.index, - split + bundle, entry.index, split ); trace!( - " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", - spill, - empty_lr - ); + " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", + spill, empty_lr + ); } break; } @@ -720,9 +712,7 @@ impl<'a, F: Function> Env<'a, F> { .unwrap(); trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - new_bundle, - entry.index, - spill + new_bundle, entry.index, spill ); self.ctx.bundles[spill].ranges.push(entry); self.ctx.bundles[new_bundle].ranges.drain(..1); @@ -760,14 +750,11 @@ impl<'a, F: Function> Env<'a, F> { }); trace!( " -> bundle {:?} range {:?}: first use implies split point {:?}", - bundle, - entry.index, - first_use, + bundle, entry.index, first_use, ); trace!( " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", - spill, - empty_lr + spill, empty_lr ); } break; @@ -790,6 +777,153 @@ impl<'a, F: Function> Env<'a, F> { } } + /// Does `bundle` have live ranges in both cold and non-cold blocks? + fn bundle_spans_hot_and_cold(&self, bundle: LiveBundleIndex) -> bool { + if !self.ctx.cfginfo.any_cold { + return false; + } + let cfg = &self.ctx.cfginfo; + let mut seen_hot = false; + let mut seen_cold = false; + for entry in &self.ctx.bundles[bundle].ranges { + let first = cfg.insn_block[entry.range.from.inst().index()].index(); + let last = cfg.insn_block[entry.range.to.prev().inst().index()].index(); + for block in first..=last { + if cfg.cold[block] { + seen_cold = true; + } else { + seen_hot = true; + } + if seen_hot && seen_cold { + return true; + } + } + } + false + } + + /// Split off the cold portion of a bundle that spans both cold and non-cold + /// blocks into its own bundle, cutting live ranges at block boundaries, and + /// requeue both halves. + /// + /// This keeps a conflict inside a cold block (e.g. the clobbers of a call + /// on a slow path) from forcing the hot portion of the bundle to be split + /// or spilled: the cold portion is allocated on its own, and any spills and + /// reloads it needs stay in cold code. + /// + /// Splits nothing and returns `false` if the bundle does not cover both hot + /// and cold blocks. + /// + /// This does not count towards `MAX_SPLITS_PER_SPILLSET`. It terminates + /// because both resulting bundles are entirely hot or entirely cold, and + /// further splits only produce subsets of them. + pub fn try_split_off_cold(&mut self, bundle: LiveBundleIndex, hint: PReg) -> bool { + if !self.bundle_spans_hot_and_cold(bundle) { + return false; + } + + trace!("splitting cold portion off of bundle {bundle:?}"); + self.ctx.output.stats.splits_cold += 1; + + let empty = LiveRangeList::new_in(self.ctx.bump()); + let ranges = core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty); + let mut hot = LiveRangeList::new_in(self.ctx.bump()); + let mut cold = LiveRangeList::new_in(self.ctx.bump()); + + for entry in &ranges { + let end = entry.range.to; + let mut lr = entry.index; + loop { + let from = self.ctx.ranges[lr].range.from; + let block = self.ctx.cfginfo.insn_block[from.inst().index()].index(); + let is_cold = self.ctx.cfginfo.cold[block]; + + // Find the end of the run of blocks with the same coldness. + let mut next = block + 1; + while next < self.ctx.cfginfo.block_entry.len() + && self.ctx.cfginfo.block_entry[next] < end + && self.ctx.cfginfo.cold[next] == is_cold + { + next += 1; + } + + // The cut point, if the run didn't span to the end of the + // range. + let cut = if next < self.ctx.cfginfo.block_entry.len() + && self.ctx.cfginfo.block_entry[next] < end + { + Some(self.ctx.cfginfo.block_entry[next]) + } else { + None + }; + + // Split everything after the run off into a new LR. + let rest = cut.map(|cut| { + let new_lr = self.ctx.ranges.add( + CodeRange { from: cut, to: end }, + self.ctx.scratch_bump.clone(), + ); + let vreg = self.ctx.ranges[lr].vreg; + self.ctx.ranges[new_lr].vreg = vreg; + let first_use = self.ctx.ranges[lr] + .uses + .iter() + .position(|u| u.pos >= cut) + .unwrap_or(self.ctx.ranges[lr].uses.len()); + let mut rest_uses = UseList::new_in(self.ctx.bump()); + rest_uses.extend(self.ctx.ranges[lr].uses.iter().cloned().skip(first_use)); + self.ctx.ranges[new_lr].uses = rest_uses; + self.ctx.ranges[lr].uses.truncate(first_use); + self.ctx.ranges[lr].range.to = cut; + self.recompute_range_properties(lr); + self.recompute_range_properties(new_lr); + self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { + range: self.ctx.ranges[new_lr].range, + index: new_lr, + }); + new_lr + }); + + let piece = LiveRangeListEntry { + range: self.ctx.ranges[lr].range, + index: lr, + }; + if is_cold { + cold.push(piece); + } else { + hot.push(piece); + } + + match rest { + Some(rest) => lr = rest, + None => break, + } + } + } + + debug_assert!(!hot.is_empty() && !cold.is_empty()); + + let spillset = self.ctx.bundles[bundle].spillset; + let cold_bundle = self.ctx.bundles.add(self.ctx.scratch_bump.clone()); + self.ctx.bundles[cold_bundle].spillset = spillset; + for entry in &cold { + self.ctx.ranges[entry.index].bundle = cold_bundle; + } + self.ctx.bundles[cold_bundle].ranges = cold; + + for entry in &hot { + self.ctx.ranges[entry.index].bundle = bundle; + } + self.ctx.bundles[bundle].ranges = hot; + + for b in [bundle, cold_bundle] { + self.recompute_bundle_properties(b); + let prio = self.ctx.bundles[b].prio; + self.ctx.allocation_queue.insert(b, prio as usize, hint); + } + true + } + /// Splits the given bundle into minimal bundles per Use, falling /// back onto the spill bundle. This must work for any bundle no /// matter how many conflicts. @@ -905,9 +1039,7 @@ impl<'a, F: Function> Env<'a, F> { trace!( " -> created new LR {:?} range {:?} with new bundle {:?} for this use", - lr, - cr, - new_bundle + lr, cr, new_bundle ); } @@ -935,9 +1067,7 @@ impl<'a, F: Function> Env<'a, F> { self.ctx.ranges[spill_lr].bundle = spill; trace!( " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", - spill_range, - spill_lr, - spill + spill_range, spill_lr, spill ); } else { assert!(spill_uses.is_empty()); @@ -1066,6 +1196,7 @@ impl<'a, F: Function> Env<'a, F> { let mut lowest_cost_split_conflict_cost: Option = None; let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0)); let mut lowest_cost_split_conflict_reg = PReg::invalid(); + let mut cold_fixed_conflict = false; // Heuristic: start the scan for an available // register at an offset influenced both by our @@ -1117,8 +1248,7 @@ impl<'a, F: Function> Env<'a, F> { AllocRegResult::Conflict(bundles, first_conflict_point) => { trace!( " -> conflict with bundles {:?}, first conflict at {:?}", - bundles, - first_conflict_point + bundles, first_conflict_point ); let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles); @@ -1150,7 +1280,13 @@ impl<'a, F: Function> Env<'a, F> { } } AllocRegResult::ConflictWithFixed(max_cost, point) => { - trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); + trace!( + " -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", + max_cost, point + ); + + let point_block = self.ctx.cfginfo.insn_block[point.inst().index()]; + cold_fixed_conflict |= self.ctx.cfginfo.cold[point_block.index()]; let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth( self.ctx.cfginfo.insn_block[point.inst().index()], @@ -1185,8 +1321,7 @@ impl<'a, F: Function> Env<'a, F> { trace!( " -> lowest cost evict: set {:?}, cost {:?}", - lowest_cost_evict_conflict_set, - lowest_cost_evict_conflict_cost, + lowest_cost_evict_conflict_set, lowest_cost_evict_conflict_cost, ); trace!( " -> lowest cost split: cost {:?}, point {:?}, reg {:?}", @@ -1277,16 +1412,16 @@ impl<'a, F: Function> Env<'a, F> { } trace!( " -> total {}, fixed {}, min {}", - total_regs, - fixed_assigned, - min_bundles_assigned + total_regs, fixed_assigned, min_bundles_assigned ); if min_bundles_assigned + fixed_assigned >= total_regs { return Err(RegAllocError::TooManyLiveRegs); } } - panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve"); + panic!( + "Could not allocate minimal bundle, but the allocation problem should be possible to solve" + ); } // If our bundle's weight is less than or equal to(*) the @@ -1316,6 +1451,14 @@ impl<'a, F: Function> Env<'a, F> { core::cmp::max(lowest_cost_split_conflict_point, bundle_start); let requeue_with_reg = lowest_cost_split_conflict_reg; + // If a fixed reservation (e.g. a call's clobbers) in cold + // code is in the way, first split off the cold portion of + // the bundle, so that the conflict is resolved, and any + // resulting spills and reloads placed, within cold code. + if cold_fixed_conflict && self.try_split_off_cold(bundle, requeue_with_reg) { + break 'outer; + } + // Adjust `split_at_point` if it is within a deeper loop // than the bundle start -- hoist it to just before the // first loop header it encounters. From af55e1d2ef9285f2bb6260ae30c8fab4e2be59fd Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 16 Nov 2026 08:53:09 -0800 Subject: [PATCH 4/4] cargo fmt --- src/ion/process.rs | 62 ++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/src/ion/process.rs b/src/ion/process.rs index b6de1c9f..7bd14479 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -13,21 +13,21 @@ //! Main allocation loop that processes bundles. use super::{ - Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, LiveRangeKey, - LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, Requirement, SpillWeight, - UseList, VRegIndex, spill_weight_from_constraint, + spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, + LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, + Requirement, SpillWeight, UseList, VRegIndex, }; use crate::{ - Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, - RegAllocError, ion::data_structures::{ - BUNDLE_MAX_NORMAL_SPILL_WEIGHT, CodeRange, MAX_SPLITS_PER_SPILLSET, + CodeRange, Use, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET, MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, - MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT, Use, + MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT, }, + Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, + RegAllocError, }; use core::fmt::Debug; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum AllocRegResult<'a> { @@ -214,7 +214,8 @@ impl<'a, F: Function> Env<'a, F> { pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { trace!( "evicting bundle {:?}: alloc {:?}", - bundle, self.ctx.bundles[bundle].allocation + bundle, + self.ctx.bundles[bundle].allocation ); let preg = match self.ctx.bundles[bundle].allocation.as_reg() { Some(preg) => preg, @@ -347,7 +348,8 @@ impl<'a, F: Function> Env<'a, F> { let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio; trace!( " -> dividing by prio {}; final weight {}", - self.ctx.bundles[bundle].prio, final_weight + self.ctx.bundles[bundle].prio, + final_weight ); core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight) } else { @@ -447,7 +449,8 @@ impl<'a, F: Function> Env<'a, F> { trace!( "bundle_start = {:?} bundle_end = {:?}", - bundle_start, bundle_end + bundle_start, + bundle_end ); // Is the bundle at most spanning one instruction? Then @@ -657,7 +660,9 @@ impl<'a, F: Function> Env<'a, F> { .unwrap(); trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - bundle, entry.index, spill + bundle, + entry.index, + spill ); self.ctx.bundles[spill].ranges.push(entry); self.ctx.bundles[bundle].ranges.pop(); @@ -690,7 +695,9 @@ impl<'a, F: Function> Env<'a, F> { }); trace!( " -> bundle {:?} range {:?}: last use implies split point {:?}", - bundle, entry.index, split + bundle, + entry.index, + split ); trace!( " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", @@ -712,7 +719,9 @@ impl<'a, F: Function> Env<'a, F> { .unwrap(); trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - new_bundle, entry.index, spill + new_bundle, + entry.index, + spill ); self.ctx.bundles[spill].ranges.push(entry); self.ctx.bundles[new_bundle].ranges.drain(..1); @@ -750,11 +759,14 @@ impl<'a, F: Function> Env<'a, F> { }); trace!( " -> bundle {:?} range {:?}: first use implies split point {:?}", - bundle, entry.index, first_use, + bundle, + entry.index, + first_use, ); trace!( " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", - spill, empty_lr + spill, + empty_lr ); } break; @@ -1039,7 +1051,9 @@ impl<'a, F: Function> Env<'a, F> { trace!( " -> created new LR {:?} range {:?} with new bundle {:?} for this use", - lr, cr, new_bundle + lr, + cr, + new_bundle ); } @@ -1067,7 +1081,9 @@ impl<'a, F: Function> Env<'a, F> { self.ctx.ranges[spill_lr].bundle = spill; trace!( " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", - spill_range, spill_lr, spill + spill_range, + spill_lr, + spill ); } else { assert!(spill_uses.is_empty()); @@ -1248,7 +1264,8 @@ impl<'a, F: Function> Env<'a, F> { AllocRegResult::Conflict(bundles, first_conflict_point) => { trace!( " -> conflict with bundles {:?}, first conflict at {:?}", - bundles, first_conflict_point + bundles, + first_conflict_point ); let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles); @@ -1321,7 +1338,8 @@ impl<'a, F: Function> Env<'a, F> { trace!( " -> lowest cost evict: set {:?}, cost {:?}", - lowest_cost_evict_conflict_set, lowest_cost_evict_conflict_cost, + lowest_cost_evict_conflict_set, + lowest_cost_evict_conflict_cost, ); trace!( " -> lowest cost split: cost {:?}, point {:?}, reg {:?}", @@ -1412,7 +1430,9 @@ impl<'a, F: Function> Env<'a, F> { } trace!( " -> total {}, fixed {}, min {}", - total_regs, fixed_assigned, min_bundles_assigned + total_regs, + fixed_assigned, + min_bundles_assigned ); if min_bundles_assigned + fixed_assigned >= total_regs { return Err(RegAllocError::TooManyLiveRegs);