diff --git a/src/cfg.rs b/src/cfg.rs index 499ab835..6cd90627 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -34,6 +34,10 @@ pub struct CFGInfo { /// indices. Otherwise, it will be approximate, but should still /// be usable for heuristic purposes. pub approx_loop_depth: Vec, + /// Whether each block is cold. + pub cold: Vec, + /// Is any block cold? + pub any_cold: bool, } impl CFGInfo { @@ -63,6 +67,8 @@ impl CFGInfo { .block_exit .repopulate(nb, ProgPoint::before(Inst::invalid())); let (backedge_in, backedge_out) = ctx.backedge.repopulate(nb * 2, 0).split_at_mut(nb); + let cold = self.cold.repopulate(nb, false); + let mut any_cold = false; for block in 0..f.num_blocks() { let block = Block::new(block); @@ -71,6 +77,9 @@ impl CFGInfo { } block_entry[block.index()] = ProgPoint::before(f.block_insns(block).first()); block_exit[block.index()] = ProgPoint::after(f.block_insns(block).last()); + let block_is_cold = f.is_block_cold(block); + cold[block.index()] = block_is_cold; + any_cold |= block_is_cold; // Check critical edge condition: if there is more than // one predecessor, each must have only one successor @@ -113,6 +122,8 @@ impl CFGInfo { } } + self.any_cold = any_cold; + let approx_loop_depth = self.approx_loop_depth.cleared(); let mut backedge_stack: SmallVec<[u32; 4]> = smallvec![]; let mut cur_depth = 0; @@ -136,4 +147,16 @@ impl CFGInfo { Ok(()) } + + /// The loop depth to use when weighing spill costs in `block`. + /// + /// Cold blocks are weighted as if they were outside of every loop, so that + /// their uses don't make hot code pay to keep values in registers. + pub fn spill_weight_loop_depth(&self, block: Block) -> u32 { + if self.cold[block.index()] { + 0 + } else { + self.approx_loop_depth[block.index()] + } + } } diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index 6bc8d067..cd7f870d 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -57,6 +57,7 @@ pub struct Func { num_vregs: usize, reftype_vregs: Vec, debug_value_labels: Vec<(VReg, Inst, Inst, u32)>, + cold_blocks: Vec, } impl Function for Func { @@ -129,6 +130,10 @@ impl Function for Func { RegClass::Vector => 2, } } + + fn is_block_cold(&self, block: Block) -> bool { + self.cold_blocks[block.index()] + } } struct FuncBuilder { @@ -153,6 +158,7 @@ impl FuncBuilder { num_vregs: 0, reftype_vregs: vec![], debug_value_labels: vec![], + cold_blocks: vec![], }, insts_per_block: vec![], } @@ -167,6 +173,7 @@ impl FuncBuilder { self.f.block_succs.push(vec![]); self.f.block_params_in.push(vec![]); self.f.block_params_out.push(vec![]); + self.f.cold_blocks.push(false); self.insts_per_block.push(vec![]); b } @@ -370,6 +377,7 @@ pub struct Options { pub fixed_def_clobbers: bool, pub reftypes: bool, pub callsite_ish_constraints: bool, + pub cold_blocks: bool, pub num_blocks: RangeInclusive, pub num_vregs_per_block: RangeInclusive, pub num_uses_per_inst: RangeInclusive, @@ -387,6 +395,7 @@ impl Options { fixed_def_clobbers: false, reftypes: false, callsite_ish_constraints: false, + cold_blocks: false, num_blocks: 1..=100, num_vregs_per_block: 5..=15, num_uses_per_inst: 0..=10, @@ -449,6 +458,12 @@ impl Func { } builder.compute_doms(); + if opts.cold_blocks { + for block in 0..num_blocks { + builder.f.cold_blocks[block] = u.ratio(1, 4)?; + } + } + let alloc_vreg = |builder: &mut FuncBuilder, u: &mut Unstructured| { let vreg = VReg::new(builder.f.num_vregs, RegClass::arbitrary(u)?); builder.f.num_vregs += 1; @@ -703,10 +718,11 @@ impl core::fmt::Debug for Func { }) .collect::>() .join(", "); + let cold = if self.cold_blocks[i] { " cold" } else { "" }; write!( f, - " block{}({}): # succs:{:?} preds:{:?}\n", - i, params_in, succs, preds + " block{}({}){}: # succs:{:?} preds:{:?}\n", + i, params_in, cold, succs, preds )?; for inst in blockrange.iter() { write!( diff --git a/src/fuzzing/ion.rs b/src/fuzzing/ion.rs index c76037cd..5228b14a 100644 --- a/src/fuzzing/ion.rs +++ b/src/fuzzing/ion.rs @@ -14,6 +14,7 @@ const OPTIONS: func::Options = func::Options { fixed_def_clobbers: true, reftypes: true, callsite_ish_constraints: true, + cold_blocks: true, ..func::Options::DEFAULT }; diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index ba036a11..d8adbee4 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -968,6 +968,7 @@ pub struct Stats { pub splits_conflicts: usize, pub splits_defs: usize, pub splits_all: usize, + pub splits_cold: usize, pub final_liverange_count: usize, pub final_bundle_count: usize, pub spill_bundle_count: usize, diff --git a/src/ion/liveranges.rs b/src/ion/liveranges.rs index 09040f6c..3de12be7 100644 --- a/src/ion/liveranges.rs +++ b/src/ion/liveranges.rs @@ -216,7 +216,7 @@ impl<'a, F: Function> Env<'a, F> { let operand = u.operand; let constraint = operand.constraint(); let block = self.cfginfo.insn_block[u.pos.inst().index()]; - let loop_depth = self.cfginfo.approx_loop_depth[block.index()] as usize; + let loop_depth = self.cfginfo.spill_weight_loop_depth(block) as usize; let weight = spill_weight_from_constraint( constraint, loop_depth, diff --git a/src/ion/process.rs b/src/ion/process.rs index 57b81e69..7bd14479 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -700,10 +700,9 @@ impl<'a, F: Function> Env<'a, F> { split ); trace!( - " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", - spill, - empty_lr - ); + " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", + spill, empty_lr + ); } break; } @@ -790,6 +789,153 @@ impl<'a, F: Function> Env<'a, F> { } } + /// Does `bundle` have live ranges in both cold and non-cold blocks? + fn bundle_spans_hot_and_cold(&self, bundle: LiveBundleIndex) -> bool { + if !self.ctx.cfginfo.any_cold { + return false; + } + let cfg = &self.ctx.cfginfo; + let mut seen_hot = false; + let mut seen_cold = false; + for entry in &self.ctx.bundles[bundle].ranges { + let first = cfg.insn_block[entry.range.from.inst().index()].index(); + let last = cfg.insn_block[entry.range.to.prev().inst().index()].index(); + for block in first..=last { + if cfg.cold[block] { + seen_cold = true; + } else { + seen_hot = true; + } + if seen_hot && seen_cold { + return true; + } + } + } + false + } + + /// Split off the cold portion of a bundle that spans both cold and non-cold + /// blocks into its own bundle, cutting live ranges at block boundaries, and + /// requeue both halves. + /// + /// This keeps a conflict inside a cold block (e.g. the clobbers of a call + /// on a slow path) from forcing the hot portion of the bundle to be split + /// or spilled: the cold portion is allocated on its own, and any spills and + /// reloads it needs stay in cold code. + /// + /// Splits nothing and returns `false` if the bundle does not cover both hot + /// and cold blocks. + /// + /// This does not count towards `MAX_SPLITS_PER_SPILLSET`. It terminates + /// because both resulting bundles are entirely hot or entirely cold, and + /// further splits only produce subsets of them. + pub fn try_split_off_cold(&mut self, bundle: LiveBundleIndex, hint: PReg) -> bool { + if !self.bundle_spans_hot_and_cold(bundle) { + return false; + } + + trace!("splitting cold portion off of bundle {bundle:?}"); + self.ctx.output.stats.splits_cold += 1; + + let empty = LiveRangeList::new_in(self.ctx.bump()); + let ranges = core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty); + let mut hot = LiveRangeList::new_in(self.ctx.bump()); + let mut cold = LiveRangeList::new_in(self.ctx.bump()); + + for entry in &ranges { + let end = entry.range.to; + let mut lr = entry.index; + loop { + let from = self.ctx.ranges[lr].range.from; + let block = self.ctx.cfginfo.insn_block[from.inst().index()].index(); + let is_cold = self.ctx.cfginfo.cold[block]; + + // Find the end of the run of blocks with the same coldness. + let mut next = block + 1; + while next < self.ctx.cfginfo.block_entry.len() + && self.ctx.cfginfo.block_entry[next] < end + && self.ctx.cfginfo.cold[next] == is_cold + { + next += 1; + } + + // The cut point, if the run didn't span to the end of the + // range. + let cut = if next < self.ctx.cfginfo.block_entry.len() + && self.ctx.cfginfo.block_entry[next] < end + { + Some(self.ctx.cfginfo.block_entry[next]) + } else { + None + }; + + // Split everything after the run off into a new LR. + let rest = cut.map(|cut| { + let new_lr = self.ctx.ranges.add( + CodeRange { from: cut, to: end }, + self.ctx.scratch_bump.clone(), + ); + let vreg = self.ctx.ranges[lr].vreg; + self.ctx.ranges[new_lr].vreg = vreg; + let first_use = self.ctx.ranges[lr] + .uses + .iter() + .position(|u| u.pos >= cut) + .unwrap_or(self.ctx.ranges[lr].uses.len()); + let mut rest_uses = UseList::new_in(self.ctx.bump()); + rest_uses.extend(self.ctx.ranges[lr].uses.iter().cloned().skip(first_use)); + self.ctx.ranges[new_lr].uses = rest_uses; + self.ctx.ranges[lr].uses.truncate(first_use); + self.ctx.ranges[lr].range.to = cut; + self.recompute_range_properties(lr); + self.recompute_range_properties(new_lr); + self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { + range: self.ctx.ranges[new_lr].range, + index: new_lr, + }); + new_lr + }); + + let piece = LiveRangeListEntry { + range: self.ctx.ranges[lr].range, + index: lr, + }; + if is_cold { + cold.push(piece); + } else { + hot.push(piece); + } + + match rest { + Some(rest) => lr = rest, + None => break, + } + } + } + + debug_assert!(!hot.is_empty() && !cold.is_empty()); + + let spillset = self.ctx.bundles[bundle].spillset; + let cold_bundle = self.ctx.bundles.add(self.ctx.scratch_bump.clone()); + self.ctx.bundles[cold_bundle].spillset = spillset; + for entry in &cold { + self.ctx.ranges[entry.index].bundle = cold_bundle; + } + self.ctx.bundles[cold_bundle].ranges = cold; + + for entry in &hot { + self.ctx.ranges[entry.index].bundle = bundle; + } + self.ctx.bundles[bundle].ranges = hot; + + for b in [bundle, cold_bundle] { + self.recompute_bundle_properties(b); + let prio = self.ctx.bundles[b].prio; + self.ctx.allocation_queue.insert(b, prio as usize, hint); + } + true + } + /// Splits the given bundle into minimal bundles per Use, falling /// back onto the spill bundle. This must work for any bundle no /// matter how many conflicts. @@ -1066,6 +1212,7 @@ impl<'a, F: Function> Env<'a, F> { let mut lowest_cost_split_conflict_cost: Option = None; let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0)); let mut lowest_cost_split_conflict_reg = PReg::invalid(); + let mut cold_fixed_conflict = false; // Heuristic: start the scan for an available // register at an offset influenced both by our @@ -1131,10 +1278,9 @@ impl<'a, F: Function> Env<'a, F> { lowest_cost_evict_conflict_set.extend(bundles); } - let loop_depth = - self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block - [first_conflict_point.inst().index()] - .index()]; + let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth( + self.ctx.cfginfo.insn_block[first_conflict_point.inst().index()], + ); let move_cost = spill_weight_from_constraint( OperandConstraint::Reg, loop_depth as usize, @@ -1151,10 +1297,17 @@ impl<'a, F: Function> Env<'a, F> { } } AllocRegResult::ConflictWithFixed(max_cost, point) => { - trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); + trace!( + " -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", + max_cost, point + ); - let loop_depth = self.ctx.cfginfo.approx_loop_depth - [self.ctx.cfginfo.insn_block[point.inst().index()].index()]; + let point_block = self.ctx.cfginfo.insn_block[point.inst().index()]; + cold_fixed_conflict |= self.ctx.cfginfo.cold[point_block.index()]; + + let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth( + self.ctx.cfginfo.insn_block[point.inst().index()], + ); let move_cost = spill_weight_from_constraint( OperandConstraint::Reg, loop_depth as usize, @@ -1286,7 +1439,9 @@ impl<'a, F: Function> Env<'a, F> { } } - panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve"); + panic!( + "Could not allocate minimal bundle, but the allocation problem should be possible to solve" + ); } // If our bundle's weight is less than or equal to(*) the @@ -1316,6 +1471,14 @@ impl<'a, F: Function> Env<'a, F> { core::cmp::max(lowest_cost_split_conflict_point, bundle_start); let requeue_with_reg = lowest_cost_split_conflict_reg; + // If a fixed reservation (e.g. a call's clobbers) in cold + // code is in the way, first split off the cold portion of + // the bundle, so that the conflict is resolved, and any + // resulting spills and reloads placed, within cold code. + if cold_fixed_conflict && self.try_split_off_cold(bundle, requeue_with_reg) { + break 'outer; + } + // Adjust `split_at_point` if it is within a deeper loop // than the bundle start -- hoist it to just before the // first loop header it encounters. diff --git a/src/lib.rs b/src/lib.rs index 5eab7d38..15cf8073 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1352,6 +1352,15 @@ pub trait Function { fn allow_multiple_vreg_defs(&self) -> bool { false } + + /// Is the given block cold, i.e. rarely executed? + /// + /// The allocator may use this as a hint to keep spills, reloads, and + /// other moves out of hot code and in cold blocks instead. It does not + /// affect correctness. + fn is_block_cold(&self, _block: Block) -> bool { + false + } } /// A position before or after an instruction at which we can make an diff --git a/src/serialize.rs b/src/serialize.rs index e98d8831..8ee46865 100644 --- a/src/serialize.rs +++ b/src/serialize.rs @@ -50,6 +50,8 @@ pub struct SerializableFunction { spillslot_size: Vec, multi_spillslot_named_by_last_slot: bool, allow_multiple_vreg_defs: bool, + #[serde(default)] + cold_blocks: Vec, } impl SerializableFunction { @@ -119,6 +121,9 @@ impl SerializableFunction { .to_vec(), multi_spillslot_named_by_last_slot: func.multi_spillslot_named_by_last_slot(), allow_multiple_vreg_defs: func.allow_multiple_vreg_defs(), + cold_blocks: (0..func.num_blocks()) + .map(|i| func.is_block_cold(Block::new(i))) + .collect(), } } @@ -196,6 +201,13 @@ impl Function for SerializableFunction { fn allow_multiple_vreg_defs(&self) -> bool { self.allow_multiple_vreg_defs } + + fn is_block_cold(&self, block: Block) -> bool { + self.cold_blocks + .get(block.index()) + .copied() + .unwrap_or(false) + } } impl fmt::Debug for SerializableFunction { @@ -255,9 +267,14 @@ impl fmt::Debug for SerializableFunction { }) .collect::>() .join(", "); + let cold = if self.is_block_cold(Block::new(i)) { + " cold" + } else { + "" + }; write!( f, - " block{i}({params_in}): # succs:{succs:?} preds:{preds:?}\n", + " block{i}({params_in}){cold}: # succs:{succs:?} preds:{preds:?}\n", )?; for inst in blockrange.iter() { let ops: Vec<_> = self