--- 4.2.2/src/core/column.rs +++ candidate/src/core/column.rs @@ -791,7 +791,7 @@ for pred_idx in 0..target_idx { let pred_rows = &parsed_columns[pred_idx]; if pred_rows.len() == target_rows.len() && target_rows.len() >= 16 { - if let Some(cand) = evaluate_inter_column(pred_rows, target_rows, pred_idx) { + if let Some(cand) = evaluate_inter_column(pred_rows, target_rows, pred_idx, best_m10.as_ref().map_or(current_best_len, Vec::len)) { let cand_len = cand.len(); let is_better = best_m10.as_ref().map_or(cand_len < current_best_len, |b| cand_len < b.len()); if is_better { @@ -816,6 +816,7 @@ predictor: &[&[u8]], target: &[&[u8]], pred_idx: usize, + budget: usize, ) -> Option> { let records = predictor.len(); if records != target.len() || records < 16 { @@ -845,14 +846,28 @@ return None; } - // Build joint distribution + // Every distinct predictor must appear in the mapping, with its encoded + // length and bytes plus at least one target-length varint. These costs can + // only increase, so a candidate reaching the incumbent budget cannot win. + let fixed_cost = 1 + varint_len(records) + varint_len(pred_idx) + + 1 + (records + 7) / 8; // Exception count needs at least one byte. + if fixed_cost + 1 >= budget { return None; } + let mut mapping_lower_bound = 0usize; + let mut distinct = 0usize; let mut joint_counts: HashMap<&[u8], HashMap<&[u8], usize>> = HashMap::new(); for i in 0..records { - *joint_counts - .entry(predictor[i]) - .or_default() - .entry(target[i]) - .or_insert(0) += 1; + let targets = match joint_counts.entry(predictor[i]) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + distinct += 1; + mapping_lower_bound += varint_len(predictor[i].len()) + predictor[i].len() + 1; + if fixed_cost + varint_len(distinct) + mapping_lower_bound >= budget { + return None; + } + entry.insert(HashMap::new()) + } + }; + *targets.entry(target[i]).or_insert(0) += 1; } let mut mapping: Vec<(&[u8], &[u8])> = Vec::with_capacity(joint_counts.len()); @@ -901,7 +916,7 @@ out.extend_from_slice(exc); } - Some(out) + if out.len() < budget { Some(out) } else { None } } /// Adds column mode tags; stream kinds 3/4/7/8/11 distinguish them from legacy kinds 1/2/5/6/9. @@ -1913,10 +1928,32 @@ predictor.extend(std::iter::repeat(b"tie".as_slice()).take(32)); let mut target=vec![b"fixed".as_slice();60]; for _ in 0..16 {target.extend([b"a-long-target".as_slice(),b"z".as_slice()]);} - let first=evaluate_inter_column(&predictor,&target,0).unwrap(); + let first=evaluate_inter_column(&predictor,&target,0,usize::MAX).unwrap(); // Every invocation creates fresh randomized hash tables. for _ in 0..64 { - assert_eq!(evaluate_inter_column(&predictor,&target,0).unwrap(),first); - } - } -} + assert_eq!(evaluate_inter_column(&predictor,&target,0,usize::MAX).unwrap(),first); + } + } +} + +#[cfg(test)] +mod dependency_budget_tests { + use super::evaluate_inter_column; + #[test] + fn exact_budget_preserves_candidate_bytes_and_strict_ties() { + for records in [16,64,128,129,1024] { + for cardinality in [1,2,7,127,128,129] { + let predictor:Vec<_>=(0..records).map(|i|format!("predictor-{}",i%cardinality)).collect(); + let target:Vec<_>=(0..records).map(|i|format!("target-{}",i%cardinality)).collect(); + let p:Vec<_>=predictor.iter().map(|s|s.as_bytes()).collect(); + let t:Vec<_>=target.iter().map(|s|s.as_bytes()).collect(); + let expected=evaluate_inter_column(&p,&t,128,usize::MAX).unwrap(); + for budget in [0,1,expected.len()-1,expected.len(),expected.len()+1,usize::MAX] { + let bounded=evaluate_inter_column(&p,&t,128,budget); + if budget>expected.len() {assert_eq!(bounded,Some(expected.clone()));} + else {assert_eq!(bounded,None);} + } + } + } + } +}