Check expected values of fairlyAllocateCredit with confidence intervals

Author: apasel422

impl/package-lock.json

@@ -17,6 +17,7 @@
     "html-webpack-plugin": "^5.6.3",
     "http-server": "^14.1.1",
     "prettier": "^3.6.2",
+    "simple-statistics": "^7.8.8",
     "ts-loader": "^9.5.2",
     "typescript": "^5.9.2",
     "typescript-eslint": "^8.39.0",

impl/package.json

@@ -2,6 +2,7 @@ import { fairlyAllocateCredit } from "./backend";
 
 import { strict as assert } from "assert";
 import test from "node:test";
+import { inverseErrorFunction } from "simple-statistics";
 
 interface FairlyAllocateCreditTestCase {
   name: string;
@@ -14,17 +15,49 @@ function noRand(): number {
   throw new Error("no rand expected");
 }
 
-// TODO: Check that the distribution of credit arrays matches expectations.
+type Interval = [min: number, max: number];
+
+// https://en.wikipedia.org/wiki/Probit
+function normalPpf(q: number, stdDev: number): number {
+  return stdDev * Math.sqrt(2) * inverseErrorFunction(2 * q - 1);
+}
+
+const minNForIntervalApprox = 1000;
+
+function getIntervalApprox(n: number, p: number, alpha: number): Interval {
+  if (n < minNForIntervalApprox) {
+    throw new RangeError(`n must be >= ${minNForIntervalApprox}`);
+  }
+
+  // Approximates a binomial distribution with a normal distribution which is a bit
+  // simpler as it is symmetric.
+  const mean = n * p;
+  const variance = mean * (1 - p);
+  const diff = normalPpf(1 - alpha / 2, Math.sqrt(variance));
+  return [mean - diff, mean + diff];
+}
+
+function getAllIntervals(
+  n: number,
+  creditFractions: readonly number[],
+  alphaTotal: number,
+): Interval[] {
+  // We are testing one hypothesis per dimension, so divide `alphaTotal` by
+  // the number of dimensions: https://en.wikipedia.org/wiki/Bonferroni_correction
+  const alpha = alphaTotal / creditFractions.length;
+  return creditFractions.map((cf) => getIntervalApprox(n, cf, alpha));
+}
+
 function runFairlyAllocateCreditTest(
   tc: Readonly<FairlyAllocateCreditTestCase>,
 ): void {
   // TODO: replace with precise sum
   const sumCredit = tc.credit.reduce((a, b) => a + b, 0);
-  const normalizedFloatCredit = tc.credit.map(
-    (item) => (tc.value * item) / sumCredit,
-  );
+  const normalizedFloatCredit = tc.credit.map((item) => item / sumCredit);
+
+  const [rand, k] = tc.needsRand ? [Math.random, 1000] : [noRand, 1];
 
-  const [rand, k] = tc.needsRand ? [Math.random, 10000] : [noRand, 1];
+  const totals = new Array<number>(tc.credit.length).fill(0);
 
   for (let n = 0; n < k; ++n) {
     const actualCredit = fairlyAllocateCredit(tc.credit, tc.value, rand);
@@ -34,12 +67,14 @@ function runFairlyAllocateCreditTest(
     for (const [j, actual] of actualCredit.entries()) {
       assert.ok(Number.isInteger(actual));
 
-      const normalized = normalizedFloatCredit[j]!;
+      const normalized = normalizedFloatCredit[j]! * tc.value;
       const diff = Math.abs(actual - normalized);
       assert.ok(
-        diff <= 1,
-        `credit error > 1: actual=${actual}, normalized=${normalized}`,
+        diff < 1,
+        `credit error >= 1: actual=${actual}, normalized=${normalized}`,
       );
+
+      totals[j]! += actual / tc.value;
     }
 
     assert.equal(
@@ -49,6 +84,25 @@ function runFairlyAllocateCreditTest(
       `actual credit does not sum to value: ${actualCredit.join(", ")}`,
     );
   }
+
+  const alpha = 0.00001; // Probability of test failing at random.
+
+  const intervals: Interval[] =
+    k > 1
+      ? getAllIntervals(
+          k,
+          normalizedFloatCredit.map((c) => c - Math.floor(c)),
+          alpha,
+        )
+      : normalizedFloatCredit.map((c) => [c, c]);
+
+  for (const [j, total] of totals.entries()) {
+    const [min, max] = intervals[j]!;
+    assert.ok(
+      total >= min && total <= max,
+      `total for credit[${j}] ${total} not in ${1 - alpha} confidence interval [${min}, ${max}]`,
+    );
+  }
 }
 
 const testCases: FairlyAllocateCreditTestCase[] = [