What is Sample Ratio Mismatch (SRM)? Detection & Prevention

-- Calculate SRM for a 50/50 split experiment
WITH experiment_counts AS (
  SELECT
    variant,
    COUNT(*) as observed,
    SUM(COUNT(*)) OVER () as total
  FROM experiment_assignments
  WHERE experiment_id = 'your-experiment-id'
  GROUP BY variant
),
expected AS (
  SELECT
    variant,
    observed,
    total * 0.5 as expected  -- 50% expected per variant
  FROM experiment_counts
),
chi_squared AS (
  SELECT
    SUM(POWER(observed - expected, 2) / expected) as chi_sq,
    COUNT(*) - 1 as df  -- degrees of freedom
  FROM expected
)
SELECT
  chi_sq,
  -- p-value approximation (for df=1)
  -- If chi_sq > 10.83, p < 0.001 (SRM detected)
  CASE
    WHEN chi_sq > 10.83 THEN 'SRM DETECTED - DO NOT TRUST RESULTS'
    WHEN chi_sq > 6.63 THEN 'WARNING - Possible SRM (p < 0.01)'
    ELSE 'OK - No SRM detected'
  END as srm_status
FROM chi_squared;

from scipy import stats

def detect_srm(observed_a: int, observed_b: int, expected_ratio: float = 0.5) -> dict:
    """
    Detect Sample Ratio Mismatch using chi-squared test.
    
    Args:
        observed_a: Number of users in variant A
        observed_b: Number of users in variant B
        expected_ratio: Expected ratio for variant A (default 0.5 for 50/50)
    
    Returns:
        dict with chi_squared, p_value, and srm_detected
    """
    total = observed_a + observed_b
    expected_a = total * expected_ratio
    expected_b = total * (1 - expected_ratio)
    
    chi_sq, p_value = stats.chisquare(
        f_obs=[observed_a, observed_b],
        f_exp=[expected_a, expected_b]
    )
    
    return {
        'chi_squared': chi_sq,
        'p_value': p_value,
        'srm_detected': p_value < 0.001,
        'observed_ratio': observed_a / total,
        'expected_ratio': expected_ratio
    }

# Example usage
result = detect_srm(observed_a=5500, observed_b=4500)
print(f"SRM Detected: {result['srm_detected']}")
print(f"P-value: {result['p_value']:.6f}")
print(f"Observed ratio: {result['observed_ratio']:.2%}")