Skip to main content

fastcit_core/ci_tests/
pillai.rs

1//! Pillai's-trace conditional-independence test (mixed discrete/continuous
2//! data), after pgmpy's `PillaiTrace` with linear residualization.
3//!
4//! X and Y are linearly regressed on `[1, Z]` (a continuous variable gives one
5//! residual column; a discrete variable with K levels gives K−1 columns, a
6//! linear probability model). With residual blocks `R_x` (n × p) and `R_y`
7//! (n × q), `s = min(p, q)`, Pillai's trace is
8//! `V = Σ ρᵢ² = tr(S_xx⁻¹ S_xy S_yy⁻¹ S_yx)` over the canonical correlations,
9//! computed exactly (no iterative CCA). The p-value uses pgmpy's
10//! F-approximation, `df1 = p·q`, `df2 = s·(n − 1 + s − p − q)`,
11//! `F = (V/df1)·(df2/(s − V))`, evaluated with the F survival function. The
12//! effect size is `V/s`; the statistic is `V`.
13//!
14//! As in pgmpy, `df2` ignores the number of conditioning columns. When `n`
15//! minus the number of design columns (`[1, Z]`) equals `max(p, q)` and `Z` is
16//! not empty, the residuals force `V = s` and the test returns `p = 0`
17//! ("dependent"), exactly as pgmpy does. When it is smaller than `max(p, q)`,
18//! a residual block is singular and the test returns an error. Linearly
19//! dependent conditioning columns (duplicate or nested categoricals, a constant
20//! column) raise a degenerate-data error, whereas pgmpy fits them by least
21//! squares.
22
23use faer::{Mat, MatRef};
24use statrs::distribution::{ContinuousCDF, FisherSnedecor};
25
26use crate::dataset::Dataset;
27use crate::error::CIError;
28use crate::residual::{design_matrix, ols_residuals_multi, target_block};
29use crate::strategy::{CIResult, CITest, DataType, IndependenceRule, TestMeta};
30
31/// Relative threshold below which a residual column is treated as zero.
32const RESIDUAL_TOL: f64 = 1e-10;
33/// Smallest admissible squared Cholesky pivot of a unit-diagonal block.
34const PIVOT_TOL: f64 = 1e-10;
35/// `V` within this relative distance of `s` counts as perfect dependence.
36const PERFECT_TOL: f64 = 1e-12;
37
38/// Pillai's-trace test (linear residualization, mixed data).
39#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
40pub struct Pillai;
41
42impl Pillai {
43    /// Construct the test (no configuration).
44    #[must_use]
45    pub fn new() -> Self {
46        Self
47    }
48}
49
50/// Residual block of `col` on `design`, with a check that no residual column
51/// is (numerically) constant zero relative to its target column.
52fn residual_block(data: &Dataset, design: &Mat<f64>, col: usize) -> Result<Mat<f64>, CIError> {
53    let label = data
54        .name_of(col)
55        .map_or_else(|| col.to_string(), |n| format!("`{n}`"));
56    if let Ok(values) = data.continuous(col) {
57        let (lo, hi) = values
58            .iter()
59            .fold((f64::INFINITY, f64::NEG_INFINITY), |(l, h), &v| {
60                (l.min(v), h.max(v))
61            });
62        if lo >= hi {
63            return Err(CIError::DegenerateData(format!(
64                "column {label} is constant"
65            )));
66        }
67    }
68    let target = target_block(data, col)?;
69    let res = ols_residuals_multi(design.as_ref(), target.as_ref())?;
70    let n = target.nrows();
71    #[allow(clippy::cast_precision_loss)]
72    let nf = n as f64;
73    for j in 0..target.ncols() {
74        let mean = (0..n).map(|i| target[(i, j)]).sum::<f64>() / nf;
75        let scale: f64 = (0..n).map(|i| (target[(i, j)] - mean).powi(2)).sum();
76        let max_abs = (0..n).fold(0.0_f64, |m, i| m.max(target[(i, j)].abs()));
77        let res_ss: f64 = (0..n).map(|i| res[(i, j)].powi(2)).sum();
78        let noise = nf * f64::EPSILON * f64::EPSILON * max_abs * max_abs;
79        if !res_ss.is_finite() || res_ss <= RESIDUAL_TOL * RESIDUAL_TOL * scale + noise {
80            return Err(CIError::DegenerateData(format!(
81                "residual of column {label} given Z has no variance"
82            )));
83        }
84    }
85    Ok(res)
86}
87
88/// Column-centers `m` and scales every column to unit Euclidean norm.
89fn standardize(m: MatRef<'_, f64>) -> Result<Mat<f64>, CIError> {
90    let (n, k) = (m.nrows(), m.ncols());
91    #[allow(clippy::cast_precision_loss)]
92    let nf = n as f64;
93    let mut out = Mat::zeros(n, k);
94    for j in 0..k {
95        let mean = (0..n).map(|i| m[(i, j)]).sum::<f64>() / nf;
96        let norm = (0..n)
97            .map(|i| (m[(i, j)] - mean).powi(2))
98            .sum::<f64>()
99            .sqrt();
100        if !(norm.is_finite() && norm > 0.0) {
101            return Err(CIError::DegenerateData(
102                "residual column has no variance".to_string(),
103            ));
104        }
105        for i in 0..n {
106            out[(i, j)] = (m[(i, j)] - mean) / norm;
107        }
108    }
109    Ok(out)
110}
111
112/// `aᵀ b` for two matrices with the same number of rows.
113fn cross(a: MatRef<'_, f64>, b: MatRef<'_, f64>) -> Mat<f64> {
114    Mat::from_fn(a.ncols(), b.ncols(), |i, j| {
115        (0..a.nrows()).map(|r| a[(r, i)] * b[(r, j)]).sum()
116    })
117}
118
119/// Lower Cholesky factor of a unit-diagonal correlation matrix; a pivot whose
120/// square is not above [`PIVOT_TOL`] means the block is (numerically) singular.
121#[allow(clippy::many_single_char_names)] // standard linear-algebra names
122fn cholesky(c: &Mat<f64>) -> Result<Mat<f64>, CIError> {
123    let k = c.nrows();
124    let mut l = Mat::<f64>::zeros(k, k);
125    for j in 0..k {
126        let d = c[(j, j)] - (0..j).map(|t| l[(j, t)].powi(2)).sum::<f64>();
127        if !(d.is_finite() && d > PIVOT_TOL) {
128            return Err(CIError::DegenerateData(
129                "residual block is singular (collinear or determined by Z)".to_string(),
130            ));
131        }
132        let ljj = d.sqrt();
133        l[(j, j)] = ljj;
134        for i in j + 1..k {
135            let s: f64 = (0..j).map(|t| l[(i, t)] * l[(j, t)]).sum();
136            l[(i, j)] = (c[(i, j)] - s) / ljj;
137        }
138    }
139    Ok(l)
140}
141
142/// Solves `L W = B` (forward substitution) for lower-triangular `L`.
143fn forward_solve(l: &Mat<f64>, b: &Mat<f64>) -> Mat<f64> {
144    let mut w = Mat::<f64>::zeros(b.nrows(), b.ncols());
145    for c in 0..b.ncols() {
146        for i in 0..b.nrows() {
147            let s: f64 = (0..i).map(|t| l[(i, t)] * w[(t, c)]).sum();
148            w[(i, c)] = (b[(i, c)] - s) / l[(i, i)];
149        }
150    }
151    w
152}
153
154/// Pillai's trace `tr(S_xx⁻¹ S_xy S_yy⁻¹ S_yx)` of two residual blocks.
155///
156/// # Errors
157///
158/// Returns [`CIError::DegenerateData`] if either block is singular or a
159/// column is constant, or [`CIError::DimensionMismatch`] on unequal row counts.
160pub(crate) fn pillai_trace(rx: MatRef<'_, f64>, ry: MatRef<'_, f64>) -> Result<f64, CIError> {
161    if rx.nrows() != ry.nrows() {
162        return Err(CIError::DimensionMismatch(format!(
163            "residual blocks have {} and {} rows",
164            rx.nrows(),
165            ry.nrows()
166        )));
167    }
168    let ux = standardize(rx)?;
169    let uy = standardize(ry)?;
170    let lx = cholesky(&cross(ux.as_ref(), ux.as_ref()))?;
171    let ly = cholesky(&cross(uy.as_ref(), uy.as_ref()))?;
172    // M = Lx⁻¹ Cxy Ly⁻ᵀ ; V = ‖M‖_F². Compute Lx⁻¹ Cxy, then solve for Ly.
173    let a = forward_solve(&lx, &cross(ux.as_ref(), uy.as_ref()));
174    // Rows of `a` times Ly⁻ᵀ: solve Ly Wᵀ = aᵀ.
175    let at = Mat::from_fn(a.ncols(), a.nrows(), |i, j| a[(j, i)]);
176    let w = forward_solve(&ly, &at);
177    let v: f64 = (0..w.nrows())
178        .map(|i| (0..w.ncols()).map(|j| w[(i, j)].powi(2)).sum::<f64>())
179        .sum();
180    if !v.is_finite() {
181        return Err(CIError::DegenerateData(
182            "Pillai's trace is not finite".to_string(),
183        ));
184    }
185    Ok(v)
186}
187
188#[allow(clippy::many_single_char_names, clippy::similar_names)] // x, y, z, n, p, q, s are the standard names
189impl CITest for Pillai {
190    fn test_impl(
191        &self,
192        data: &Dataset,
193        x: usize,
194        y: usize,
195        z: &[usize],
196    ) -> Result<CIResult, CIError> {
197        let design = design_matrix(data, z)?;
198        let rx = residual_block(data, &design, x)?;
199        let ry = residual_block(data, &design, y)?;
200        let (n, p, q) = (rx.nrows(), rx.ncols(), ry.ncols());
201        let s = p.min(q);
202        let v = pillai_trace(rx.as_ref(), ry.as_ref())?;
203
204        #[allow(clippy::cast_precision_loss)]
205        let (sf, df1, df2) = {
206            let sf = s as f64;
207            let df1 = (p * q) as f64;
208            // pgmpy: df2 = s · (n − 1 + s − p − q), ignoring |Z|.
209            let df2 = sf * ((n + s) as f64 - 1.0 - (p + q) as f64);
210            (sf, df1, df2)
211        };
212        if df2.is_nan() || df2 <= 0.0 {
213            return Err(CIError::DegenerateData(format!(
214                "denominator degrees of freedom {df2} are not positive"
215            )));
216        }
217        let v = v.min(sf);
218        let p_value = if sf - v <= PERFECT_TOL * sf {
219            0.0
220        } else {
221            let f_stat = (v / df1) * (df2 / (sf - v));
222            let dist = FisherSnedecor::new(df1, df2)
223                .map_err(|e| CIError::Numeric(format!("F distribution: {e}")))?;
224            dist.sf(f_stat)
225        };
226        Ok(CIResult {
227            statistic: Some(v),
228            p_value,
229            dof: None,
230            effect_size: Some(v / sf),
231        })
232    }
233
234    fn meta(&self) -> TestMeta {
235        TestMeta {
236            name: "pillai",
237            data_types: &[DataType::Discrete, DataType::Continuous],
238            symmetric: true,
239            rule: IndependenceRule::PValueGe,
240        }
241    }
242}
243
244#[cfg(test)]
245#[allow(
246    clippy::cast_possible_truncation,
247    clippy::cast_precision_loss,
248    clippy::cast_sign_loss,
249    clippy::float_cmp,
250    clippy::many_single_char_names,
251    clippy::similar_names,
252    clippy::unreadable_literal
253)] // test helpers: small n, explicit math
254mod tests {
255    use super::*;
256    use crate::ci_tests::PearsonCorrelation;
257    use crate::dataset::ColumnKind;
258
259    fn lcg(n: usize, seed: u64) -> Vec<f64> {
260        let mut s = seed;
261        (0..n)
262            .map(|_| {
263                s = s
264                    .wrapping_mul(6364136223846793005)
265                    .wrapping_add(1442695040888963407);
266                let v = (s >> 11) as f64 / (1u64 << 53) as f64;
267                v - 0.5
268            })
269            .collect()
270    }
271
272    fn ds(cs: Vec<(&str, ColumnKind, Vec<f64>)>) -> Dataset {
273        Dataset::from_columns(
274            cs.into_iter()
275                .map(|(n, k, v)| (n.to_string(), k, v))
276                .collect(),
277        )
278        .unwrap()
279    }
280
281    fn levels(u: &[f64], k: usize) -> Vec<f64> {
282        u.iter()
283            .map(|v| (((v + 0.5) * k as f64) as usize).min(k - 1) as f64)
284            .collect()
285    }
286
287    #[test]
288    fn continuous_pair_matches_squared_pearson() {
289        let n = 50;
290        let (e, f) = (lcg(n, 1), lcg(n, 2));
291        let x = e.clone();
292        let y: Vec<f64> = (0..n).map(|i| 0.6 * e[i] + f[i]).collect();
293        let data = ds(vec![
294            ("x", ColumnKind::Continuous, x.clone()),
295            ("y", ColumnKind::Continuous, y.clone()),
296        ]);
297        let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
298        let mx = x.iter().sum::<f64>() / n as f64;
299        let my = y.iter().sum::<f64>() / n as f64;
300        let sxy: f64 = (0..n).map(|i| (x[i] - mx) * (y[i] - my)).sum();
301        let sxx: f64 = x.iter().map(|a| (a - mx).powi(2)).sum();
302        let syy: f64 = y.iter().map(|a| (a - my).powi(2)).sum();
303        let r2 = sxy * sxy / (sxx * syy);
304        assert!((r.statistic.unwrap() - r2).abs() < 1e-12);
305        assert!((r.effect_size.unwrap() - r2).abs() < 1e-12);
306        let pc = PearsonCorrelation::new().test(&data, 0, 1, &[]).unwrap();
307        assert!((r.p_value - pc.p_value).abs() < 1e-10);
308        assert_eq!(r.dof, None);
309    }
310
311    #[test]
312    fn categorical_vs_continuous_is_one_way_anova() {
313        let n = 60;
314        let k = 3;
315        let x = levels(&lcg(n, 5), k);
316        let noise = lcg(n, 6);
317        let y: Vec<f64> = (0..n).map(|i| 0.4 * x[i] + noise[i]).collect();
318        let data = ds(vec![
319            ("x", ColumnKind::Discrete, x.clone()),
320            ("y", ColumnKind::Continuous, y.clone()),
321        ]);
322        let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
323        let grand = y.iter().sum::<f64>() / n as f64;
324        let (mut ssb, mut ssw) = (0.0, 0.0);
325        for g in 0..k {
326            let ys: Vec<f64> = (0..n)
327                .filter(|&i| x[i] as usize == g)
328                .map(|i| y[i])
329                .collect();
330            let m = ys.iter().sum::<f64>() / ys.len() as f64;
331            ssb += ys.len() as f64 * (m - grand).powi(2);
332            ssw += ys.iter().map(|v| (v - m).powi(2)).sum::<f64>();
333        }
334        let v = ssb / (ssb + ssw);
335        assert!((r.statistic.unwrap() - v).abs() < 1e-12);
336        let f_anova = (ssb / (k - 1) as f64) / (ssw / (n - k) as f64);
337        let p = FisherSnedecor::new((k - 1) as f64, (n - k) as f64)
338            .unwrap()
339            .sf(f_anova);
340        assert!((r.p_value - p).abs() < 1e-10);
341    }
342
343    #[test]
344    fn mixed_conditioning_runs_and_is_symmetric() {
345        let n = 120;
346        let x = levels(&lcg(n, 7), 3);
347        let zd = levels(&lcg(n, 8), 2);
348        let zc = lcg(n, 9);
349        let e = lcg(n, 10);
350        let y: Vec<f64> = (0..n).map(|i| 0.3 * x[i] + zd[i] - zc[i] + e[i]).collect();
351        let data = ds(vec![
352            ("x", ColumnKind::Discrete, x),
353            ("y", ColumnKind::Continuous, y),
354            ("zd", ColumnKind::Discrete, zd),
355            ("zc", ColumnKind::Continuous, zc),
356        ]);
357        let a = Pillai::new().test(&data, 0, 1, &[2, 3]).unwrap();
358        let b = Pillai::new().test(&data, 1, 0, &[2, 3]).unwrap();
359        assert!(a.p_value.is_finite() && (0.0..=1.0).contains(&a.p_value));
360        assert!((a.statistic.unwrap() - b.statistic.unwrap()).abs() < 1e-12);
361        assert!((a.p_value - b.p_value).abs() < 1e-12);
362    }
363
364    #[test]
365    fn perfect_dependence_gives_p_zero() {
366        let n = 40;
367        let x = lcg(n, 11);
368        let data = ds(vec![
369            ("x", ColumnKind::Continuous, x.clone()),
370            ("y", ColumnKind::Continuous, x),
371        ]);
372        let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
373        assert!((r.statistic.unwrap() - 1.0).abs() < 1e-12);
374        assert_eq!(r.p_value, 0.0);
375
376        let cx = levels(&lcg(n, 12), 3);
377        let cy: Vec<f64> = cx.iter().map(|v| (v + 1.0) % 3.0).collect();
378        let data = ds(vec![
379            ("x", ColumnKind::Discrete, cx),
380            ("y", ColumnKind::Discrete, cy),
381        ]);
382        let r = Pillai::new().test(&data, 0, 1, &[]).unwrap();
383        assert!((r.statistic.unwrap() - 2.0).abs() < 1e-12);
384        assert!(r.statistic.unwrap().is_finite());
385        assert_eq!(r.p_value, 0.0);
386    }
387
388    #[test]
389    fn categorical_x_determined_by_z_is_degenerate() {
390        let n = 60;
391        let zd = levels(&lcg(n, 13), 3);
392        let x: Vec<f64> = zd.iter().map(|v| (v + 1.0) % 3.0).collect();
393        let y = lcg(n, 14);
394        let data = ds(vec![
395            ("x", ColumnKind::Discrete, x),
396            ("y", ColumnKind::Continuous, y),
397            ("z", ColumnKind::Discrete, zd),
398        ]);
399        assert!(matches!(
400            Pillai::new().test(&data, 0, 1, &[2]),
401            Err(CIError::DegenerateData(_))
402        ));
403    }
404
405    #[test]
406    fn constant_continuous_x_or_y_is_degenerate() {
407        let n = 50;
408        let (a, b) = (lcg(n, 21), lcg(n, 22));
409        for c in [0.1, 1.0, 3.7] {
410            for z in [vec![], vec![2usize]] {
411                for const_is_y in [true, false] {
412                    let k = vec![c; n];
413                    let (xs, ys) = if const_is_y {
414                        (a.clone(), k)
415                    } else {
416                        (k, a.clone())
417                    };
418                    let data = ds(vec![
419                        ("x", ColumnKind::Continuous, xs),
420                        ("y", ColumnKind::Continuous, ys),
421                        ("z", ColumnKind::Continuous, b.clone()),
422                    ]);
423                    assert!(
424                        matches!(
425                            Pillai::new().test(&data, 0, 1, &z),
426                            Err(CIError::DegenerateData(_))
427                        ),
428                        "c={c} z={z:?} const_is_y={const_is_y}"
429                    );
430                }
431            }
432        }
433    }
434
435    #[test]
436    fn f_tail_is_accurate() {
437        let p = FisherSnedecor::new(3.0, 200.0).unwrap().sf(40.0);
438        let expected = 2.7079201317013815e-20;
439        assert!(((p - expected) / expected).abs() < 1e-9, "{p}");
440    }
441
442    fn scaled_data(n: usize, scales: &[f64]) -> Dataset {
443        let z1 = lcg(n, 11);
444        let z2 = lcg(n, 12);
445        let x: Vec<f64> = (0..n).map(|i| z1[i] + lcg(n, 13)[i]).collect();
446        let y: Vec<f64> = (0..n).map(|i| z1[i] - z2[i] + lcg(n, 14)[i]).collect();
447        let (s1, s2) = (scales[0], scales[1]);
448        ds(vec![
449            ("x", ColumnKind::Continuous, x),
450            ("y", ColumnKind::Continuous, y),
451            (
452                "z1",
453                ColumnKind::Continuous,
454                z1.iter().map(|v| v * s1).collect(),
455            ),
456            (
457                "z2",
458                ColumnKind::Continuous,
459                z2.iter().map(|v| v * s2).collect(),
460            ),
461        ])
462    }
463
464    #[test]
465    fn conditioning_scale_does_not_change_results() {
466        use crate::ci_tests::Gcm;
467        let base = scaled_data(60, &[1.0, 1.0]);
468        let pb = Pillai::new().test(&base, 0, 1, &[2, 3]).unwrap();
469        let gb = Gcm::new().test(&base, 0, 1, &[2, 3]).unwrap();
470        for sc in [[1e12, 1e12], [1e-12, 1e-12], [1e6, 1e-5], [1e11, 1.0]] {
471            let d = scaled_data(60, &sc);
472            let p = Pillai::new().test(&d, 0, 1, &[2, 3]).unwrap();
473            let g = Gcm::new().test(&d, 0, 1, &[2, 3]).unwrap();
474            let rel = |a: f64, b: f64| (a - b).abs() / b.abs().max(1e-300);
475            assert!(
476                rel(p.statistic.unwrap(), pb.statistic.unwrap()) < 1e-9,
477                "{sc:?}"
478            );
479            assert!(rel(p.p_value, pb.p_value) < 1e-9, "{sc:?}");
480            assert!(
481                rel(g.statistic.unwrap(), gb.statistic.unwrap()) < 1e-9,
482                "{sc:?}"
483            );
484            assert!(rel(g.p_value, gb.p_value) < 1e-9, "{sc:?}");
485        }
486    }
487
488    #[test]
489    fn constant_or_duplicated_z_still_rejected_and_tiny_variance_target_accepted() {
490        use crate::ci_tests::Gcm;
491        let n = 40;
492        let (x, y, z) = (lcg(n, 21), lcg(n, 22), lcg(n, 23));
493        let cd = |extra: Vec<f64>| {
494            ds(vec![
495                ("x", ColumnKind::Continuous, x.clone()),
496                ("y", ColumnKind::Continuous, y.clone()),
497                ("z", ColumnKind::Continuous, z.clone()),
498                ("w", ColumnKind::Continuous, extra),
499            ])
500        };
501        for scale in [1.0, 1e12, 1e-12] {
502            for extra in [vec![scale; n], z.iter().map(|v| v * scale).collect()] {
503                let d = cd(extra);
504                assert!(matches!(
505                    Pillai::new().test(&d, 0, 1, &[2, 3]),
506                    Err(CIError::DegenerateData(_))
507                ));
508                assert!(matches!(
509                    Gcm::new().test(&d, 0, 1, &[2, 3]),
510                    Err(CIError::DegenerateData(_))
511                ));
512            }
513        }
514        let xs: Vec<f64> = x.iter().map(|v| 1.0 + 1e-9 * v).collect();
515        let d = ds(vec![
516            ("x", ColumnKind::Continuous, xs),
517            ("y", ColumnKind::Continuous, y.clone()),
518            ("z", ColumnKind::Continuous, z.clone()),
519        ]);
520        assert!(Pillai::new().test(&d, 0, 1, &[2]).is_ok());
521        assert!(Gcm::new().test(&d, 0, 1, &[2]).is_ok());
522    }
523
524    #[test]
525    fn errors_use_column_names() {
526        let n = 30;
527        let (y, z) = (lcg(n, 31), lcg(n, 32));
528        let d = ds(vec![
529            ("alpha", ColumnKind::Continuous, vec![0.5; n]),
530            ("beta", ColumnKind::Continuous, y.clone()),
531            ("z", ColumnKind::Continuous, z.clone()),
532        ]);
533        let msg = Pillai::new().test(&d, 0, 1, &[]).unwrap_err().to_string();
534        assert!(msg.contains("alpha"), "{msg}");
535        let x: Vec<f64> = z.iter().map(|v| 2.0 * v + 1.0).collect();
536        let d = ds(vec![
537            ("alpha", ColumnKind::Continuous, x),
538            ("beta", ColumnKind::Continuous, y),
539            ("z", ColumnKind::Continuous, z),
540        ]);
541        let msg = Pillai::new().test(&d, 0, 1, &[2]).unwrap_err().to_string();
542        assert!(msg.contains("alpha"), "{msg}");
543    }
544}