Check expected values of fairlyAllocateCredit with confidence intervals

Author: apasel422

impl/package-lock.json

@@ -17,6 +17,7 @@
     "html-webpack-plugin": "^5.6.3",
     "http-server": "^14.1.1",
     "prettier": "^3.6.2",
+    "simple-statistics": "^7.8.8",
     "ts-loader": "^9.5.2",
     "typescript": "^5.9.2",
     "typescript-eslint": "^8.39.0",

impl/package.json

@@ -2,6 +2,7 @@ import { fairlyAllocateCredit } from "./backend";
 
 import { strict as assert } from "assert";
 import test from "node:test";
+import * as simplestatistics from "simple-statistics";
 
 interface FairlyAllocateCreditTestCase {
   name: string;
@@ -14,7 +15,32 @@ function noRand(): number {
   throw new Error("no rand expected");
 }
 
-// TODO: Check that the distribution of credit arrays matches expectations.
+type Interval = [min: number, max: number];
+
+// https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
+function ppf(q: number, mean: number, stdDev: number): number {
+  return (
+    mean +
+    stdDev * Math.sqrt(2) * simplestatistics.inverseErrorFunction(2 * q - 1)
+  );
+}
+
+function getIntervalApprox(n: number, p: number, alpha: number): Interval {
+  const mean = n * p;
+  const variance = mean * (1 - p);
+  const diff = ppf(1 - alpha / 2, mean, Math.sqrt(variance)) - mean;
+  return [mean - diff, mean + diff];
+}
+
+function getAllIntervals(
+  n: number,
+  creditFractions: readonly number[],
+  alphaTotal: number,
+): Interval[] {
+  const alpha = alphaTotal / creditFractions.length;
+  return creditFractions.map((cf) => getIntervalApprox(n, cf, alpha));
+}
+
 function runFairlyAllocateCreditTest(
   tc: Readonly<FairlyAllocateCreditTestCase>,
 ): void {
@@ -24,7 +50,9 @@ function runFairlyAllocateCreditTest(
     (item) => (tc.value * item) / sumCredit,
   );
 
-  const [rand, k] = tc.needsRand ? [Math.random, 10000] : [noRand, 1];
+  const [rand, k] = tc.needsRand ? [Math.random, 100000] : [noRand, 1];
+
+  const totals = new Array<number>(tc.credit.length).fill(0);
 
   for (let n = 0; n < k; ++n) {
     const actualCredit = fairlyAllocateCredit(tc.credit, tc.value, rand);
@@ -40,6 +68,8 @@ function runFairlyAllocateCreditTest(
         diff <= 1,
         `credit error > 1: actual=${actual}, normalized=${normalized}`,
       );
+
+      totals[j]! += actual / tc.value;
     }
 
     assert.equal(
@@ -49,6 +79,19 @@ function runFairlyAllocateCreditTest(
       `actual credit does not sum to value: ${actualCredit.join(", ")}`,
     );
   }
+
+  const intervals = getAllIntervals(
+    k,
+    tc.credit.map((c) => c / sumCredit),
+    0.0001,
+  );
+  for (const [j, total] of totals.entries()) {
+    const [min, max] = intervals[j]!;
+    assert.ok(
+      total >= min && total <= max,
+      `total for credit[${j}] ${total} not in confidence interval [${min}, ${max}]`,
+    );
+  }
 }
 
 const testCases: FairlyAllocateCreditTestCase[] = [