1use faer::{Mat, MatRef};
24use statrs::distribution::{ContinuousCDF, FisherSnedecor};
25
26use crate::dataset::Dataset;
27use crate::error::CIError;
28use crate::residual::{design_matrix, ols_residuals_multi, target_block};
29use crate::strategy::{CIResult, CITest, DataType, IndependenceRule, TestMeta};
30
31const RESIDUAL_TOL: f64 = 1e-10;
33const PIVOT_TOL: f64 = 1e-10;
35const PERFECT_TOL: f64 = 1e-12;
37
38#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
40pub struct Pillai;
41
42impl Pillai {
43 #[must_use]
45 pub fn new() -> Self {
46 Self
47 }
48}
49
50fn residual_block(data: &Dataset, design: &Mat<f64>, col: usize) -> Result<Mat<f64>, CIError> {
53 let label = data
54 .name_of(col)
55 .map_or_else(|| col.to_string(), |n| format!("`{n}`"));
56 if let Ok(values) = data.continuous(col) {
57 let (lo, hi) = values
58 .iter()
59 .fold((f64::INFINITY, f64::NEG_INFINITY), |(l, h), &v| {
60 (l.min(v), h.max(v))
61 });
62 if lo >= hi {
63 return Err(CIError::DegenerateData(format!(
64 "column {label} is constant"
65 )));
66 }
67 }
68 let target = target_block(data, col)?;
69 let res = ols_residuals_multi(design.as_ref(), target.as_ref())?;
70 let n = target.nrows();
71 #[allow(clippy::cast_precision_loss)]
72 let nf = n as f64;
73 for j in 0..target.ncols() {
74 let mean = (0..n).map(|i| target[(i, j)]).sum::<f64>() / nf;
75 let scale: f64 = (0..n).map(|i| (target[(i, j)] - mean).powi(2)).sum();
76 let max_abs = (0..n).fold(0.0_f64, |m, i| m.max(target[(i, j)].abs()));
77 let res_ss: f64 = (0..n).map(|i| res[(i, j)].powi(2)).sum();
78 let noise = nf * f64::EPSILON * f64::EPSILON * max_abs * max_abs;
79 if !res_ss.is_finite() || res_ss <= RESIDUAL_TOL * RESIDUAL_TOL * scale + noise {
80 return Err(CIError::DegenerateData(format!(
81 "residual of column {label} given Z has no variance"
82 )));
83 }
84 }
85 Ok(res)
86}
87
88fn standardize(m: MatRef<'_, f64>) -> Result<Mat<f64>, CIError> {
90 let (n, k) = (m.nrows(), m.ncols());
91 #[allow(clippy::cast_precision_loss)]
92 let nf = n as f64;
93 let mut out = Mat::zeros(n, k);
94 for j in 0..k {
95 let mean = (0..n).map(|i| m[(i, j)]).sum::<f64>() / nf;
96 let norm = (0..n)
97 .map(|i| (m[(i, j)] - mean).powi(2))
98 .sum::<f64>()
99 .sqrt();
100 if !(norm.is_finite() && norm > 0.0) {
101 return Err(CIError::DegenerateData(
102 "residual column has no variance".to_string(),
103 ));
104 }
105 for i in 0..n {
106 out[(i, j)] = (m[(i, j)] - mean) / norm;
107 }
108 }
109 Ok(out)
110}
111
112fn cross(a: MatRef<'_, f64>, b: MatRef<'_, f64>) -> Mat<f64> {
114 Mat::from_fn(a.ncols(), b.ncols(), |i, j| {
115 (0..a.nrows()).map(|r| a[(r, i)] * b[(r, j)]).sum()
116 })
117}
118
119#[allow(clippy::many_single_char_names)] fn cholesky(c: &Mat<f64>) -> Result<Mat<f64>, CIError> {
123 let k = c.nrows();
124 let mut l = Mat::<f64>::zeros(k, k);
125 for j in 0..k {
126 let d = c[(j, j)] - (0..j).map(|t| l[(j, t)].powi(2)).sum::<f64>();
127 if !(d.is_finite() && d > PIVOT_TOL) {
128 return Err(CIError::DegenerateData(
129 "residual block is singular (collinear or determined by Z)".to_string(),
130 ));
131 }
132 let ljj = d.sqrt();
133 l[(j, j)] = ljj;
134 for i in j + 1..k {
135 let s: f64 = (0..j).map(|t| l[(i, t)] * l[(j, t)]).sum();
136 l[(i, j)] = (c[(i, j)] - s) / ljj;
137 }
138 }
139 Ok(l)
140}
141
142fn forward_solve(l: &Mat<f64>, b: &Mat<f64>) -> Mat<f64> {
144 let mut w = Mat::<f64>::zeros(b.nrows(), b.ncols());
145 for c in 0..b.ncols() {
146 for i in 0..b.nrows() {
147 let s: f64 = (0..i).map(|t| l[(i, t)] * w[(t, c)]).sum();
148 w[(i, c)] = (b[(i, c)] - s) / l[(i, i)];
149 }
150 }
151 w
152}
153
154pub(crate) fn pillai_trace(rx: MatRef<'_, f64>, ry: MatRef<'_, f64>) -> Result<f64, CIError> {
161 if rx.nrows() != ry.nrows() {
162 return Err(CIError::DimensionMismatch(format!(
163 "residual blocks have {} and {} rows",
164 rx.nrows(),
165 ry.nrows()
166 )));
167 }
168 let ux = standardize(rx)?;
169 let uy = standardize(ry)?;
170 let lx = cholesky(&cross(ux.as_ref(), ux.as_ref()))?;
171 let ly = cholesky(&cross(uy.as_ref(), uy.as_ref()))?;
172 let a = forward_solve(&lx, &cross(ux.as_ref(), uy.as_ref()));
174 let at = Mat::from_fn(a.ncols(), a.nrows(), |i, j| a[(j, i)]);
176 let w = forward_solve(&ly, &at);
177 let v: f64 = (0..w.nrows())
178 .map(|i| (0..w.ncols()).map(|j| w[(i, j)].powi(2)).sum::<f64>())
179 .sum();
180 if !v.is_finite() {
181 return Err(CIError::DegenerateData(
182 "Pillai's trace is not finite".to_string(),
183 ));
184 }
185 Ok(v)
186}
187
188#[allow(clippy::many_single_char_names, clippy::similar_names)] impl CITest for Pillai {
190 fn test_impl(
191 &self,
192 data: &Dataset,
193 x: usize,
194 y: usize,
195 z: &[usize],
196 ) -> Result<CIResult, CIError> {
197 let design = design_matrix(data, z)?;
198 let rx = residual_block(data, &design, x)?;
199 let ry = residual_block(data, &design, y)?;
200 let (n, p, q) = (rx.nrows(), rx.ncols(), ry.ncols());
201 let s = p.min(q);
202 let v = pillai_trace(rx.as_ref(), ry.as_ref())?;
203
204 #[allow(clippy::cast_precision_loss)]
205 let (sf, df1, df2) = {
206 let sf = s as f64;
207 let df1 = (p * q) as f64;
208 let df2 = sf * ((n + s) as f64 - 1.0 - (p + q) as f64);
210 (sf, df1, df2)
211 };
212 if df2.is_nan() || df2 <= 0.0 {
213 return Err(CIError::DegenerateData(format!(
214 "denominator degrees of freedom {df2} are not positive"
215 )));
216 }
217 let v = v.min(sf);
218 let p_value = if sf - v <= PERFECT_TOL * sf {
219 0.0
220 } else {
221 let f_stat = (v / df1) * (df2 / (sf - v));
222 let dist = FisherSnedecor::new(df1, df2)
223 .map_err(|e| CIError::Numeric(format!("F distribution: {e}")))?;
224 dist.sf(f_stat)
225 };
226 Ok(CIResult {
227 statistic: Some(v),
228 p_value,
229 dof: None,
230 effect_size: Some(v / sf),
231 })
232 }
233
234 fn meta(&self) -> TestMeta {
235 TestMeta {
236 name: "pillai",
237 data_types: &[DataType::Discrete, DataType::Continuous],
238 symmetric: true,
239 rule: IndependenceRule::PValueGe,
240 }
241 }
242}
243
244#[cfg(test)]
245#[allow(
246 clippy::cast_possible_truncation,
247 clippy::cast_precision_loss,
248 clippy::cast_sign_loss,
249 clippy::float_cmp,
250 clippy::many_single_char_names,
251 clippy::similar_names,
252 clippy::unreadable_literal
253)] mod tests {
255 use super::*;
256 use crate::ci_tests::PearsonCorrelation;
257 use crate::dataset::ColumnKind;
258
259 fn lcg(n: usize, seed: u64) -> Vec<f64> {
260 let mut s = seed;
261 (0..n)
262 .map(|_| {
263 s = s
264 .wrapping_mul(6364136223846793005)
265 .wrapping_add(1442695040888963407);
266 let v = (s >> 11) as f64 / (1u64 << 53) as f64;
267 v - 0.5
268 })
269 .collect()
270 }
271
272 fn ds(cs: Vec<(&str, ColumnKind, Vec<f64>)>) -> Dataset {
273 Dataset::from_columns(
274 cs.into_iter()
275 .map(|(n, k, v)| (n.to_string(), k, v))
276 .collect(),
277 )
278 .unwrap()
279 }
280
281 fn levels(u: &[f64], k: usize) -> Vec<f64> {
282 u.iter()
283 .map(|v| (((v + 0.5) * k as f64) as usize).min(k - 1) as f64)
284 .collect()
285 }
286
287 #[test]
288 fn continuous_pair_matches_squared_pearson() {
289 let n = 50;
290 let (e, f) = (lcg(n, 1), lcg(n, 2));
291 let x = e.clone();
292 let y: Vec<f64> = (0..n).map(|i| 0.6 * e[i] + f[i]).collect();
293 let data = ds(vec![
294 ("x", ColumnKind::Continuous, x.clone()),
295 ("y", ColumnKind::Continuous, y.clone()),
296 ]);
297 let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
298 let mx = x.iter().sum::<f64>() / n as f64;
299 let my = y.iter().sum::<f64>() / n as f64;
300 let sxy: f64 = (0..n).map(|i| (x[i] - mx) * (y[i] - my)).sum();
301 let sxx: f64 = x.iter().map(|a| (a - mx).powi(2)).sum();
302 let syy: f64 = y.iter().map(|a| (a - my).powi(2)).sum();
303 let r2 = sxy * sxy / (sxx * syy);
304 assert!((r.statistic.unwrap() - r2).abs() < 1e-12);
305 assert!((r.effect_size.unwrap() - r2).abs() < 1e-12);
306 let pc = PearsonCorrelation::new().test(&data, 0, 1, &[]).unwrap();
307 assert!((r.p_value - pc.p_value).abs() < 1e-10);
308 assert_eq!(r.dof, None);
309 }
310
311 #[test]
312 fn categorical_vs_continuous_is_one_way_anova() {
313 let n = 60;
314 let k = 3;
315 let x = levels(&lcg(n, 5), k);
316 let noise = lcg(n, 6);
317 let y: Vec<f64> = (0..n).map(|i| 0.4 * x[i] + noise[i]).collect();
318 let data = ds(vec![
319 ("x", ColumnKind::Discrete, x.clone()),
320 ("y", ColumnKind::Continuous, y.clone()),
321 ]);
322 let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
323 let grand = y.iter().sum::<f64>() / n as f64;
324 let (mut ssb, mut ssw) = (0.0, 0.0);
325 for g in 0..k {
326 let ys: Vec<f64> = (0..n)
327 .filter(|&i| x[i] as usize == g)
328 .map(|i| y[i])
329 .collect();
330 let m = ys.iter().sum::<f64>() / ys.len() as f64;
331 ssb += ys.len() as f64 * (m - grand).powi(2);
332 ssw += ys.iter().map(|v| (v - m).powi(2)).sum::<f64>();
333 }
334 let v = ssb / (ssb + ssw);
335 assert!((r.statistic.unwrap() - v).abs() < 1e-12);
336 let f_anova = (ssb / (k - 1) as f64) / (ssw / (n - k) as f64);
337 let p = FisherSnedecor::new((k - 1) as f64, (n - k) as f64)
338 .unwrap()
339 .sf(f_anova);
340 assert!((r.p_value - p).abs() < 1e-10);
341 }
342
343 #[test]
344 fn mixed_conditioning_runs_and_is_symmetric() {
345 let n = 120;
346 let x = levels(&lcg(n, 7), 3);
347 let zd = levels(&lcg(n, 8), 2);
348 let zc = lcg(n, 9);
349 let e = lcg(n, 10);
350 let y: Vec<f64> = (0..n).map(|i| 0.3 * x[i] + zd[i] - zc[i] + e[i]).collect();
351 let data = ds(vec![
352 ("x", ColumnKind::Discrete, x),
353 ("y", ColumnKind::Continuous, y),
354 ("zd", ColumnKind::Discrete, zd),
355 ("zc", ColumnKind::Continuous, zc),
356 ]);
357 let a = Pillai::new().test(&data, 0, 1, &[2, 3]).unwrap();
358 let b = Pillai::new().test(&data, 1, 0, &[2, 3]).unwrap();
359 assert!(a.p_value.is_finite() && (0.0..=1.0).contains(&a.p_value));
360 assert!((a.statistic.unwrap() - b.statistic.unwrap()).abs() < 1e-12);
361 assert!((a.p_value - b.p_value).abs() < 1e-12);
362 }
363
364 #[test]
365 fn perfect_dependence_gives_p_zero() {
366 let n = 40;
367 let x = lcg(n, 11);
368 let data = ds(vec![
369 ("x", ColumnKind::Continuous, x.clone()),
370 ("y", ColumnKind::Continuous, x),
371 ]);
372 let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
373 assert!((r.statistic.unwrap() - 1.0).abs() < 1e-12);
374 assert_eq!(r.p_value, 0.0);
375
376 let cx = levels(&lcg(n, 12), 3);
377 let cy: Vec<f64> = cx.iter().map(|v| (v + 1.0) % 3.0).collect();
378 let data = ds(vec![
379 ("x", ColumnKind::Discrete, cx),
380 ("y", ColumnKind::Discrete, cy),
381 ]);
382 let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
383 assert!((r.statistic.unwrap() - 2.0).abs() < 1e-12);
384 assert!(r.statistic.unwrap().is_finite());
385 assert_eq!(r.p_value, 0.0);
386 }
387
388 #[test]
389 fn categorical_x_determined_by_z_is_degenerate() {
390 let n = 60;
391 let zd = levels(&lcg(n, 13), 3);
392 let x: Vec<f64> = zd.iter().map(|v| (v + 1.0) % 3.0).collect();
393 let y = lcg(n, 14);
394 let data = ds(vec![
395 ("x", ColumnKind::Discrete, x),
396 ("y", ColumnKind::Continuous, y),
397 ("z", ColumnKind::Discrete, zd),
398 ]);
399 assert!(matches!(
400 Pillai::new().test(&data, 0, 1, &[2]),
401 Err(CIError::DegenerateData(_))
402 ));
403 }
404
405 #[test]
406 fn constant_continuous_x_or_y_is_degenerate() {
407 let n = 50;
408 let (a, b) = (lcg(n, 21), lcg(n, 22));
409 for c in [0.1, 1.0, 3.7] {
410 for z in [vec![], vec![2usize]] {
411 for const_is_y in [true, false] {
412 let k = vec![c; n];
413 let (xs, ys) = if const_is_y {
414 (a.clone(), k)
415 } else {
416 (k, a.clone())
417 };
418 let data = ds(vec![
419 ("x", ColumnKind::Continuous, xs),
420 ("y", ColumnKind::Continuous, ys),
421 ("z", ColumnKind::Continuous, b.clone()),
422 ]);
423 assert!(
424 matches!(
425 Pillai::new().test(&data, 0, 1, &z),
426 Err(CIError::DegenerateData(_))
427 ),
428 "c={c} z={z:?} const_is_y={const_is_y}"
429 );
430 }
431 }
432 }
433 }
434
435 #[test]
436 fn f_tail_is_accurate() {
437 let p = FisherSnedecor::new(3.0, 200.0).unwrap().sf(40.0);
438 let expected = 2.7079201317013815e-20;
439 assert!(((p - expected) / expected).abs() < 1e-9, "{p}");
440 }
441
442 fn scaled_data(n: usize, scales: &[f64]) -> Dataset {
443 let z1 = lcg(n, 11);
444 let z2 = lcg(n, 12);
445 let x: Vec<f64> = (0..n).map(|i| z1[i] + lcg(n, 13)[i]).collect();
446 let y: Vec<f64> = (0..n).map(|i| z1[i] - z2[i] + lcg(n, 14)[i]).collect();
447 let (s1, s2) = (scales[0], scales[1]);
448 ds(vec![
449 ("x", ColumnKind::Continuous, x),
450 ("y", ColumnKind::Continuous, y),
451 (
452 "z1",
453 ColumnKind::Continuous,
454 z1.iter().map(|v| v * s1).collect(),
455 ),
456 (
457 "z2",
458 ColumnKind::Continuous,
459 z2.iter().map(|v| v * s2).collect(),
460 ),
461 ])
462 }
463
464 #[test]
465 fn conditioning_scale_does_not_change_results() {
466 use crate::ci_tests::Gcm;
467 let base = scaled_data(60, &[1.0, 1.0]);
468 let pb = Pillai::new().test(&base, 0, 1, &[2, 3]).unwrap();
469 let gb = Gcm::new().test(&base, 0, 1, &[2, 3]).unwrap();
470 for sc in [[1e12, 1e12], [1e-12, 1e-12], [1e6, 1e-5], [1e11, 1.0]] {
471 let d = scaled_data(60, &sc);
472 let p = Pillai::new().test(&d, 0, 1, &[2, 3]).unwrap();
473 let g = Gcm::new().test(&d, 0, 1, &[2, 3]).unwrap();
474 let rel = |a: f64, b: f64| (a - b).abs() / b.abs().max(1e-300);
475 assert!(
476 rel(p.statistic.unwrap(), pb.statistic.unwrap()) < 1e-9,
477 "{sc:?}"
478 );
479 assert!(rel(p.p_value, pb.p_value) < 1e-9, "{sc:?}");
480 assert!(
481 rel(g.statistic.unwrap(), gb.statistic.unwrap()) < 1e-9,
482 "{sc:?}"
483 );
484 assert!(rel(g.p_value, gb.p_value) < 1e-9, "{sc:?}");
485 }
486 }
487
488 #[test]
489 fn constant_or_duplicated_z_still_rejected_and_tiny_variance_target_accepted() {
490 use crate::ci_tests::Gcm;
491 let n = 40;
492 let (x, y, z) = (lcg(n, 21), lcg(n, 22), lcg(n, 23));
493 let cd = |extra: Vec<f64>| {
494 ds(vec![
495 ("x", ColumnKind::Continuous, x.clone()),
496 ("y", ColumnKind::Continuous, y.clone()),
497 ("z", ColumnKind::Continuous, z.clone()),
498 ("w", ColumnKind::Continuous, extra),
499 ])
500 };
501 for scale in [1.0, 1e12, 1e-12] {
502 for extra in [vec![scale; n], z.iter().map(|v| v * scale).collect()] {
503 let d = cd(extra);
504 assert!(matches!(
505 Pillai::new().test(&d, 0, 1, &[2, 3]),
506 Err(CIError::DegenerateData(_))
507 ));
508 assert!(matches!(
509 Gcm::new().test(&d, 0, 1, &[2, 3]),
510 Err(CIError::DegenerateData(_))
511 ));
512 }
513 }
514 let xs: Vec<f64> = x.iter().map(|v| 1.0 + 1e-9 * v).collect();
515 let d = ds(vec![
516 ("x", ColumnKind::Continuous, xs),
517 ("y", ColumnKind::Continuous, y.clone()),
518 ("z", ColumnKind::Continuous, z.clone()),
519 ]);
520 assert!(Pillai::new().test(&d, 0, 1, &[2]).is_ok());
521 assert!(Gcm::new().test(&d, 0, 1, &[2]).is_ok());
522 }
523
524 #[test]
525 fn errors_use_column_names() {
526 let n = 30;
527 let (y, z) = (lcg(n, 31), lcg(n, 32));
528 let d = ds(vec![
529 ("alpha", ColumnKind::Continuous, vec![0.5; n]),
530 ("beta", ColumnKind::Continuous, y.clone()),
531 ("z", ColumnKind::Continuous, z.clone()),
532 ]);
533 let msg = Pillai::new().test(&d, 0, 1, &[]).unwrap_err().to_string();
534 assert!(msg.contains("alpha"), "{msg}");
535 let x: Vec<f64> = z.iter().map(|v| 2.0 * v + 1.0).collect();
536 let d = ds(vec![
537 ("alpha", ColumnKind::Continuous, x),
538 ("beta", ColumnKind::Continuous, y),
539 ("z", ColumnKind::Continuous, z),
540 ]);
541 let msg = Pillai::new().test(&d, 0, 1, &[2]).unwrap_err().to_string();
542 assert!(msg.contains("alpha"), "{msg}");
543 }
544}