Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/cfg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ pub struct CFGInfo {
/// indices. Otherwise, it will be approximate, but should still
/// be usable for heuristic purposes.
pub approx_loop_depth: Vec<u32>,
/// Whether each block is cold.
pub cold: Vec<bool>,
Comment thread
cfallin marked this conversation as resolved.
Resolved
/// Is any block cold?
pub any_cold: bool,
}

impl CFGInfo {
Expand Down Expand Up @@ -63,6 +67,8 @@ impl CFGInfo {
.block_exit
.repopulate(nb, ProgPoint::before(Inst::invalid()));
let (backedge_in, backedge_out) = ctx.backedge.repopulate(nb * 2, 0).split_at_mut(nb);
let cold = self.cold.repopulate(nb, false);
let mut any_cold = false;

for block in 0..f.num_blocks() {
let block = Block::new(block);
Expand All @@ -71,6 +77,9 @@ impl CFGInfo {
}
block_entry[block.index()] = ProgPoint::before(f.block_insns(block).first());
block_exit[block.index()] = ProgPoint::after(f.block_insns(block).last());
let block_is_cold = f.is_block_cold(block);
cold[block.index()] = block_is_cold;
any_cold |= block_is_cold;

// Check critical edge condition: if there is more than
// one predecessor, each must have only one successor
Expand Down Expand Up @@ -113,6 +122,8 @@ impl CFGInfo {
}
}

self.any_cold = any_cold;

let approx_loop_depth = self.approx_loop_depth.cleared();
let mut backedge_stack: SmallVec<[u32; 4]> = smallvec![];
let mut cur_depth = 0;
Expand All @@ -136,4 +147,16 @@ impl CFGInfo {

Ok(())
}

/// The loop depth to use when weighing spill costs in `block`.
///
/// Cold blocks are weighted as if they were outside of every loop, so that
/// their uses don't make hot code pay to keep values in registers.
pub fn spill_weight_loop_depth(&self, block: Block) -> u32 {
if self.cold[block.index()] {
0
} else {
self.approx_loop_depth[block.index()]
}
}
}
20 changes: 18 additions & 2 deletions src/fuzzing/func.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ pub struct Func {
num_vregs: usize,
reftype_vregs: Vec<VReg>,
debug_value_labels: Vec<(VReg, Inst, Inst, u32)>,
cold_blocks: Vec<bool>,
}

impl Function for Func {
Expand Down Expand Up @@ -129,6 +130,10 @@ impl Function for Func {
RegClass::Vector => 2,
}
}

fn is_block_cold(&self, block: Block) -> bool {
self.cold_blocks[block.index()]
}
}

struct FuncBuilder {
Expand All @@ -153,6 +158,7 @@ impl FuncBuilder {
num_vregs: 0,
reftype_vregs: vec![],
debug_value_labels: vec![],
cold_blocks: vec![],
},
insts_per_block: vec![],
}
Expand All @@ -167,6 +173,7 @@ impl FuncBuilder {
self.f.block_succs.push(vec![]);
self.f.block_params_in.push(vec![]);
self.f.block_params_out.push(vec![]);
self.f.cold_blocks.push(false);
self.insts_per_block.push(vec![]);
b
}
Expand Down Expand Up @@ -370,6 +377,7 @@ pub struct Options {
pub fixed_def_clobbers: bool,
pub reftypes: bool,
pub callsite_ish_constraints: bool,
pub cold_blocks: bool,
pub num_blocks: RangeInclusive<usize>,
pub num_vregs_per_block: RangeInclusive<usize>,
pub num_uses_per_inst: RangeInclusive<usize>,
Expand All @@ -387,6 +395,7 @@ impl Options {
fixed_def_clobbers: false,
reftypes: false,
callsite_ish_constraints: false,
cold_blocks: false,
num_blocks: 1..=100,
num_vregs_per_block: 5..=15,
num_uses_per_inst: 0..=10,
Expand Down Expand Up @@ -449,6 +458,12 @@ impl Func {
}
builder.compute_doms();

if opts.cold_blocks {
for block in 0..num_blocks {
builder.f.cold_blocks[block] = u.ratio(1, 4)?;
}
}

let alloc_vreg = |builder: &mut FuncBuilder, u: &mut Unstructured| {
let vreg = VReg::new(builder.f.num_vregs, RegClass::arbitrary(u)?);
builder.f.num_vregs += 1;
Expand Down Expand Up @@ -703,10 +718,11 @@ impl core::fmt::Debug for Func {
})
.collect::<Vec<_>>()
.join(", ");
let cold = if self.cold_blocks[i] { " cold" } else { "" };
write!(
f,
" block{}({}): # succs:{:?} preds:{:?}\n",
i, params_in, succs, preds
" block{}({}){}: # succs:{:?} preds:{:?}\n",
i, params_in, cold, succs, preds
)?;
for inst in blockrange.iter() {
write!(
Expand Down
1 change: 1 addition & 0 deletions src/fuzzing/ion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const OPTIONS: func::Options = func::Options {
fixed_def_clobbers: true,
reftypes: true,
callsite_ish_constraints: true,
cold_blocks: true,
..func::Options::DEFAULT
};

Expand Down
1 change: 1 addition & 0 deletions src/ion/data_structures.rs
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,7 @@ pub struct Stats {
pub splits_conflicts: usize,
pub splits_defs: usize,
pub splits_all: usize,
pub splits_cold: usize,
pub final_liverange_count: usize,
pub final_bundle_count: usize,
pub spill_bundle_count: usize,
Expand Down
2 changes: 1 addition & 1 deletion src/ion/liveranges.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ impl<'a, F: Function> Env<'a, F> {
let operand = u.operand;
let constraint = operand.constraint();
let block = self.cfginfo.insn_block[u.pos.inst().index()];
let loop_depth = self.cfginfo.approx_loop_depth[block.index()] as usize;
let loop_depth = self.cfginfo.spill_weight_loop_depth(block) as usize;
let weight = spill_weight_from_constraint(
constraint,
loop_depth,
Expand Down
187 changes: 175 additions & 12 deletions src/ion/process.rs
Original file line number Diff line number Diff line change
Expand Up @@ -700,10 +700,9 @@ impl<'a, F: Function> Env<'a, F> {
split
);
trace!(
" -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
spill,
empty_lr
);
" -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
spill, empty_lr
);
}
break;
}
Expand Down Expand Up @@ -790,6 +789,153 @@ impl<'a, F: Function> Env<'a, F> {
}
}

/// Does `bundle` have live ranges in both cold and non-cold blocks?
fn bundle_spans_hot_and_cold(&self, bundle: LiveBundleIndex) -> bool {
if !self.ctx.cfginfo.any_cold {
return false;
}
let cfg = &self.ctx.cfginfo;
let mut seen_hot = false;
let mut seen_cold = false;
for entry in &self.ctx.bundles[bundle].ranges {
let first = cfg.insn_block[entry.range.from.inst().index()].index();
let last = cfg.insn_block[entry.range.to.prev().inst().index()].index();
for block in first..=last {
if cfg.cold[block] {
Comment thread
cfallin marked this conversation as resolved.
Resolved
seen_cold = true;
} else {
seen_hot = true;
}
if seen_hot && seen_cold {
return true;
}
}
}
false
}

/// Split off the cold portion of a bundle that spans both cold and non-cold
/// blocks into its own bundle, cutting live ranges at block boundaries, and
/// requeue both halves.
///
/// This keeps a conflict inside a cold block (e.g. the clobbers of a call
/// on a slow path) from forcing the hot portion of the bundle to be split
/// or spilled: the cold portion is allocated on its own, and any spills and
/// reloads it needs stay in cold code.
///
/// Splits nothing and returns `false` if the bundle does not cover both hot
/// and cold blocks.
///
/// This does not count towards `MAX_SPLITS_PER_SPILLSET`. It terminates
/// because both resulting bundles are entirely hot or entirely cold, and
/// further splits only produce subsets of them.
pub fn try_split_off_cold(&mut self, bundle: LiveBundleIndex, hint: PReg) -> bool {
Comment thread
cfallin marked this conversation as resolved.
Resolved
if !self.bundle_spans_hot_and_cold(bundle) {
return false;
}

trace!("splitting cold portion off of bundle {bundle:?}");
self.ctx.output.stats.splits_cold += 1;

let empty = LiveRangeList::new_in(self.ctx.bump());
let ranges = core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty);
let mut hot = LiveRangeList::new_in(self.ctx.bump());
let mut cold = LiveRangeList::new_in(self.ctx.bump());

for entry in &ranges {
let end = entry.range.to;
let mut lr = entry.index;
loop {
let from = self.ctx.ranges[lr].range.from;
let block = self.ctx.cfginfo.insn_block[from.inst().index()].index();
let is_cold = self.ctx.cfginfo.cold[block];

// Find the end of the run of blocks with the same coldness.
let mut next = block + 1;
while next < self.ctx.cfginfo.block_entry.len()
&& self.ctx.cfginfo.block_entry[next] < end
&& self.ctx.cfginfo.cold[next] == is_cold
{
next += 1;
}

// The cut point, if the run didn't span to the end of the
// range.
let cut = if next < self.ctx.cfginfo.block_entry.len()
&& self.ctx.cfginfo.block_entry[next] < end
{
Some(self.ctx.cfginfo.block_entry[next])
} else {
None
};

// Split everything after the run off into a new LR.
let rest = cut.map(|cut| {
let new_lr = self.ctx.ranges.add(
CodeRange { from: cut, to: end },
self.ctx.scratch_bump.clone(),
);
let vreg = self.ctx.ranges[lr].vreg;
self.ctx.ranges[new_lr].vreg = vreg;
let first_use = self.ctx.ranges[lr]
.uses
.iter()
.position(|u| u.pos >= cut)
.unwrap_or(self.ctx.ranges[lr].uses.len());
let mut rest_uses = UseList::new_in(self.ctx.bump());
rest_uses.extend(self.ctx.ranges[lr].uses.iter().cloned().skip(first_use));
self.ctx.ranges[new_lr].uses = rest_uses;
self.ctx.ranges[lr].uses.truncate(first_use);
self.ctx.ranges[lr].range.to = cut;
self.recompute_range_properties(lr);
self.recompute_range_properties(new_lr);
self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
range: self.ctx.ranges[new_lr].range,
index: new_lr,
});
new_lr
});

let piece = LiveRangeListEntry {
range: self.ctx.ranges[lr].range,
index: lr,
};
if is_cold {
cold.push(piece);
} else {
hot.push(piece);
}

match rest {
Some(rest) => lr = rest,
None => break,
}
}
}

debug_assert!(!hot.is_empty() && !cold.is_empty());

let spillset = self.ctx.bundles[bundle].spillset;
let cold_bundle = self.ctx.bundles.add(self.ctx.scratch_bump.clone());
self.ctx.bundles[cold_bundle].spillset = spillset;
for entry in &cold {
self.ctx.ranges[entry.index].bundle = cold_bundle;
}
self.ctx.bundles[cold_bundle].ranges = cold;

for entry in &hot {
self.ctx.ranges[entry.index].bundle = bundle;
}
self.ctx.bundles[bundle].ranges = hot;

for b in [bundle, cold_bundle] {
self.recompute_bundle_properties(b);
let prio = self.ctx.bundles[b].prio;
self.ctx.allocation_queue.insert(b, prio as usize, hint);
}
true
}

/// Splits the given bundle into minimal bundles per Use, falling
/// back onto the spill bundle. This must work for any bundle no
/// matter how many conflicts.
Expand Down Expand Up @@ -1066,6 +1212,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut lowest_cost_split_conflict_cost: Option<u32> = None;
let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0));
let mut lowest_cost_split_conflict_reg = PReg::invalid();
let mut cold_fixed_conflict = false;

// Heuristic: start the scan for an available
// register at an offset influenced both by our
Expand Down Expand Up @@ -1131,10 +1278,9 @@ impl<'a, F: Function> Env<'a, F> {
lowest_cost_evict_conflict_set.extend(bundles);
}

let loop_depth =
self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block
[first_conflict_point.inst().index()]
.index()];
let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth(
self.ctx.cfginfo.insn_block[first_conflict_point.inst().index()],
);
let move_cost = spill_weight_from_constraint(
OperandConstraint::Reg,
loop_depth as usize,
Expand All @@ -1151,10 +1297,17 @@ impl<'a, F: Function> Env<'a, F> {
}
}
AllocRegResult::ConflictWithFixed(max_cost, point) => {
trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
trace!(
" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}",
max_cost, point
);

let loop_depth = self.ctx.cfginfo.approx_loop_depth
[self.ctx.cfginfo.insn_block[point.inst().index()].index()];
let point_block = self.ctx.cfginfo.insn_block[point.inst().index()];
cold_fixed_conflict |= self.ctx.cfginfo.cold[point_block.index()];

let loop_depth = self.ctx.cfginfo.spill_weight_loop_depth(
self.ctx.cfginfo.insn_block[point.inst().index()],
);
let move_cost = spill_weight_from_constraint(
OperandConstraint::Reg,
loop_depth as usize,
Expand Down Expand Up @@ -1286,7 +1439,9 @@ impl<'a, F: Function> Env<'a, F> {
}
}

panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve");
panic!(
"Could not allocate minimal bundle, but the allocation problem should be possible to solve"
);
}

// If our bundle's weight is less than or equal to(*) the
Expand Down Expand Up @@ -1316,6 +1471,14 @@ impl<'a, F: Function> Env<'a, F> {
core::cmp::max(lowest_cost_split_conflict_point, bundle_start);
let requeue_with_reg = lowest_cost_split_conflict_reg;

// If a fixed reservation (e.g. a call's clobbers) in cold
// code is in the way, first split off the cold portion of
// the bundle, so that the conflict is resolved, and any
// resulting spills and reloads placed, within cold code.
if cold_fixed_conflict && self.try_split_off_cold(bundle, requeue_with_reg) {
break 'outer;
}

// Adjust `split_at_point` if it is within a deeper loop
// than the bundle start -- hoist it to just before the
// first loop header it encounters.
Expand Down
Loading
Loading