added harness to support bots being able to run / test strategies.
This commit is contained in:
Jesse
+118
-13
@@ -20,7 +20,7 @@
|
||||
*/
|
||||
|
||||
import { applyIntent, areaOf, canAdvanceLoad, destinationsFor, facilityCarTypes, laborersLeft } from '../engine/apply.ts';
|
||||
import { MAX_CONSIST, nextOfficeTier } from '../engine/content.ts';
|
||||
import { MAX_CONSIST, nextOfficeTier, officeProfile } from '../engine/content.ts';
|
||||
import type { Hand, TrackGeometry } from '../engine/content.ts';
|
||||
import type { GameEvent } from '../engine/events.ts';
|
||||
import type { Intent } from '../engine/intents.ts';
|
||||
@@ -78,8 +78,42 @@ function because(reason: string, intent: Intent): Intent {
|
||||
return intent;
|
||||
}
|
||||
|
||||
export const developerBot: BotPolicy = {
|
||||
name: 'developer',
|
||||
/**
|
||||
* KNOBS FOR A/B MEASUREMENT, and nothing else.
|
||||
*
|
||||
* A heuristic change has to be measured against the bot it replaces, over the SAME deals — and
|
||||
* editing the bot between runs makes that impossible to do honestly, because the two sides of the
|
||||
* comparison never exist at once. Every flag here is off by default, so `makeDeveloperBot({})` is
|
||||
* byte-identical to the bot that came before this existed.
|
||||
*
|
||||
* TEMPORARY BY CONSTRUCTION. When a flag measures well it becomes the default and the flag is
|
||||
* deleted in the same commit; when it measures badly it is deleted with its finding recorded in the
|
||||
* changelog. What must not happen is a bot that accumulates switches nobody can account for — a
|
||||
* heuristic with no measurement attached is exactly what this machinery exists to prevent.
|
||||
*/
|
||||
export type BotTweaks = {
|
||||
/**
|
||||
* Refuse to schedule a train the Office has no room for: cap committed trains (timetable slots
|
||||
* filled, plus queued Extras) at `adTracks + trainCapSlack`.
|
||||
*
|
||||
* `undefined` means no cap — today's behaviour, which plays every train card on sight. A Whistle
|
||||
* Post has ONE A/D track, and a second train standing there makes every later arrival an
|
||||
* automatic collision (Gap 2d).
|
||||
*/
|
||||
trainCapSlack?: number;
|
||||
};
|
||||
|
||||
/** The bot as it plays today. Every knob off. */
|
||||
export const developerBot: BotPolicy = makeDeveloperBot({});
|
||||
|
||||
/** A variant, for measuring one change at a time against the bot above. */
|
||||
export function makeDeveloperBot(tweaks: BotTweaks): BotPolicy {
|
||||
const suffix = Object.entries(tweaks)
|
||||
.filter(([, v]) => v !== undefined)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(',');
|
||||
return {
|
||||
name: suffix ? `developer+${suffix}` : 'developer',
|
||||
|
||||
choose(s, player, options) {
|
||||
lastReason = 'no specific reason — first legal option';
|
||||
@@ -133,13 +167,29 @@ export const developerBot: BotPolicy = {
|
||||
|
||||
// --- Local Operations.
|
||||
if (s.clock.phase === 'localOps') {
|
||||
if (s.turn.option === null) return chooseLocalOption(s, player, options);
|
||||
return followThrough(s, player, options);
|
||||
/**
|
||||
* A CAP HAS TO BE APPLIED TO THE OPTIONS, NOT TO ONE BRANCH.
|
||||
*
|
||||
* The first attempt gated the two branches that exist to play a train card, measured exactly
|
||||
* zero difference over 400 paired seeds, and was right to: `followThrough` ends with a generic
|
||||
* "play what is in hand" fallback that played the train card anyway. Removing the option
|
||||
* itself is the only way to be sure no path reaches it.
|
||||
*
|
||||
* Never to an empty list — a bot with nothing legal to choose is a crash, and §6.2 can force
|
||||
* a play when the hand is over its limit. If filtering leaves nothing, the cap yields.
|
||||
*/
|
||||
const held = trainWouldOverfillTheOffice(s, player, tweaks)
|
||||
? options.filter((i) => !(i.type === 'card.play' && isTrainCard(s, i.cardId)))
|
||||
: options;
|
||||
const usable = held.length > 0 ? held : options;
|
||||
if (s.turn.option === null) return chooseLocalOption(s, player, usable, tweaks);
|
||||
return followThrough(s, player, usable, tweaks);
|
||||
}
|
||||
|
||||
return options[0]!;
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The one decision that matters: which of §6's three things to spend this Stage on.
|
||||
@@ -150,7 +200,37 @@ export const developerBot: BotPolicy = {
|
||||
* and spent the rest of the game stuffing green boxes whose loads could never complete for want of
|
||||
* a car to load them onto.
|
||||
*/
|
||||
function chooseLocalOption(s: GameState, player: PlayerIndex, options: Intent[]): Intent {
|
||||
/**
|
||||
* Trains this player has already committed to running: slots filled on the timetable, plus Extras
|
||||
* played and waiting for a Crew Tray.
|
||||
*
|
||||
* A Second Section is deliberately NOT counted. Its whole purpose is to put a following train into
|
||||
* an occupied Subdivision and force the Superintendent's ruling (Q9) — capping it would be capping
|
||||
* the card's reason to exist.
|
||||
*/
|
||||
function committedTrains(s: GameState): number {
|
||||
return s.timetable.filter((t) => t !== null).length + s.pendingExtras.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Would scheduling another train exceed what the Office can physically hold?
|
||||
*
|
||||
* §7 lets you play as many train cards as you draw, and a train that arrives with nowhere to stand
|
||||
* is an automatic collision (Gap 2d) — so the two rules together make a train card actively harmful
|
||||
* once the A/D tracks are spoken for. Off unless `trainCapSlack` is set.
|
||||
*/
|
||||
function trainWouldOverfillTheOffice(s: GameState, player: PlayerIndex, tweaks: BotTweaks): boolean {
|
||||
if (tweaks.trainCapSlack === undefined) return false;
|
||||
const cap = officeProfile(areaOf(s, player).tier).adTracks + tweaks.trainCapSlack;
|
||||
return committedTrains(s) >= cap;
|
||||
}
|
||||
|
||||
function chooseLocalOption(
|
||||
s: GameState,
|
||||
player: PlayerIndex,
|
||||
options: Intent[],
|
||||
tweaks: BotTweaks,
|
||||
): Intent {
|
||||
const choices = options.filter(
|
||||
(i): i is Extract<Intent, { type: 'localOps.choose' }> => i.type === 'localOps.choose',
|
||||
);
|
||||
@@ -167,7 +247,11 @@ function chooseLocalOption(s: GameState, player: PlayerIndex, options: Intent[])
|
||||
return because('an Office upgrade is in hand — more A/D track means fewer collisions', can('draw')!);
|
||||
}
|
||||
|
||||
if (can('draw') && hand.some((id) => isTrainCard(s, id))) {
|
||||
if (
|
||||
can('draw') &&
|
||||
hand.some((id) => isTrainCard(s, id)) &&
|
||||
!trainWouldOverfillTheOffice(s, player, tweaks)
|
||||
) {
|
||||
return because('a train card is in hand and its value compounds every Day', can('draw')!);
|
||||
}
|
||||
|
||||
@@ -933,7 +1017,12 @@ function moveTowardOffice(s: GameState, player: PlayerIndex, options: Intent[]):
|
||||
}
|
||||
|
||||
/** Once an option is chosen, work it to a sensible conclusion. */
|
||||
function followThrough(s: GameState, player: PlayerIndex, options: Intent[]): Intent {
|
||||
function followThrough(
|
||||
s: GameState,
|
||||
player: PlayerIndex,
|
||||
options: Intent[],
|
||||
tweaks: BotTweaks,
|
||||
): Intent {
|
||||
switch (s.turn.option) {
|
||||
case 'draw': {
|
||||
// Draw before playing — otherwise the hand empties and never refills.
|
||||
@@ -962,10 +1051,14 @@ function followThrough(s: GameState, player: PlayerIndex, options: Intent[]): In
|
||||
);
|
||||
if (upgrade) return because('upgrade the Office — a Whistle Post has ONE A/D track and a second arrival collides', upgrade);
|
||||
|
||||
// Train cards next — they take no placement and their value compounds every Day.
|
||||
const train = options.find(
|
||||
(i) => i.type === 'card.play' && i.placement === undefined && isTrainCard(s, i.cardId),
|
||||
);
|
||||
// Train cards next — they take no placement and their value compounds every Day. Unless the
|
||||
// Office cannot hold another one, in which case the card is HELD rather than played: an
|
||||
// arrival with no free A/D track is an automatic collision, and it repeats every Day.
|
||||
const train = trainWouldOverfillTheOffice(s, player, tweaks)
|
||||
? undefined
|
||||
: options.find(
|
||||
(i) => i.type === 'card.play' && i.placement === undefined && isTrainCard(s, i.cardId),
|
||||
);
|
||||
if (train) return because('a scheduled train runs every Day thereafter — the only card whose value compounds', train);
|
||||
|
||||
// Mainline modifiers rank with train cards: every train that crosses afterwards pays the
|
||||
@@ -1504,12 +1597,23 @@ export type PlayOutcome = {
|
||||
*/
|
||||
export type PlayObserver = (event: GameEvent, state: GameState) => void;
|
||||
|
||||
/**
|
||||
* Called once per DECISION, immediately before the policy is asked to choose.
|
||||
*
|
||||
* Separate from `PlayObserver` because the two sample different things. An event is something that
|
||||
* happened; a turn hook sees the position as the bot sees it, which is the only place to ask a
|
||||
* question of the form "was anything ready to be done here?" — a condition rather than an
|
||||
* occurrence. Sampling those off events instead would count them once per event in the batch.
|
||||
*/
|
||||
export type TurnObserver = (state: GameState) => void;
|
||||
|
||||
export function playGame(
|
||||
s: GameState,
|
||||
policy: BotPolicy,
|
||||
pumpFn: (s: GameState) => GameEvent[],
|
||||
maxTurns = 50_000,
|
||||
observe?: PlayObserver,
|
||||
onTurn?: TurnObserver,
|
||||
): PlayOutcome {
|
||||
const events: GameEvent[] = [];
|
||||
const intents: Intent['type'][] = [];
|
||||
@@ -1545,6 +1649,7 @@ export function playGame(
|
||||
const options = legalActions(s, actor);
|
||||
if (options.length === 0) break;
|
||||
|
||||
onTurn?.(s);
|
||||
const chosen = policy.choose(s, actor, options);
|
||||
intents.push(chosen.type);
|
||||
const r = applyIntent(s, actor, chosen);
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* Component 18b — paired A/B measurement for bot heuristics.
|
||||
*
|
||||
* Dev-side only. Runs the current bot and one tweaked variant over the SAME deals and reports the
|
||||
* per-seed difference.
|
||||
*
|
||||
* WHY PAIRED, AND WHY IT IS NOT OPTIONAL. Revenue has a standard deviation of about 9 across games,
|
||||
* so two 100-game runs of the identical bot can differ by a point through nothing but the deal. Every
|
||||
* single-change claim in the changelog before the Interlocking work is inside that noise. Giving both
|
||||
* policies the same seed removes the deal from the comparison entirely: what is left is the change.
|
||||
* Measured here, the paired difference has σ ≈ 5.3 against ≈ 9 unpaired, and at 1600 seeds the
|
||||
* standard error is ±0.13 — so a +0.5 heuristic is resolvable in under two minutes, where unpaired it
|
||||
* would need tens of thousands of games.
|
||||
*
|
||||
* WHY THE SPLIT IS PRINTED. A mean carried by a skewed tail is a different claim from a mean carried
|
||||
* by broad improvement, and only the better/worse/identical counts tell them apart. The train cap is
|
||||
* the case in point: +0.60 overall, but 130 seeds better, 145 worse and 1325 unchanged — it removes a
|
||||
* rare catastrophe rather than making the bot play better.
|
||||
*
|
||||
* WHY THE FUNNEL IS PRINTED. Revenue can rise because a channel started working or because the bot
|
||||
* abandoned an expensive channel for a cheap one, and the two are identical in a single number.
|
||||
*
|
||||
* Run with:
|
||||
* node src/sim/compare.ts 1600 trainCapSlack=1
|
||||
* node src/sim/compare.ts 400 trainCapSlack=0 --length short
|
||||
*/
|
||||
|
||||
import { pump } from '../engine/advance.ts';
|
||||
import { createGame } from '../engine/setup.ts';
|
||||
import type { GameLength } from '../engine/content.ts';
|
||||
import type { GameConfig, GameMode } from '../engine/state.ts';
|
||||
import type { BotPolicy, BotTweaks } from './bot.ts';
|
||||
import { developerBot, makeDeveloperBot, playGame } from './bot.ts';
|
||||
import type { Funnel, GameStats } from './stats.ts';
|
||||
import { funnelReport, makeFunnelProbe, summarize } from './stats.ts';
|
||||
|
||||
export type PairedResult = {
|
||||
seeds: number;
|
||||
baseline: string;
|
||||
variant: string;
|
||||
/** Per-seed revenue difference, variant minus baseline. */
|
||||
deltas: { seed: number; delta: number }[];
|
||||
mean: number;
|
||||
/** Standard error of the mean difference — the number that decides whether this is real. */
|
||||
stderr: number;
|
||||
sd: number;
|
||||
t: number;
|
||||
better: number;
|
||||
worse: number;
|
||||
identical: number;
|
||||
baseStats: GameStats[];
|
||||
variantStats: GameStats[];
|
||||
};
|
||||
|
||||
const SOLO = (length: GameLength, mode: GameMode): GameConfig => ({
|
||||
mode,
|
||||
victory: 'highestAfterDays',
|
||||
length,
|
||||
optionalRules: {
|
||||
reducedVisibility: false,
|
||||
sisterTrains: false,
|
||||
employeeRotation: false,
|
||||
emergencyToolbox: false,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* One game, one seed, one policy.
|
||||
*
|
||||
* The seed stride matches `harness.ts` exactly, so a compare run and a harness run of the same size
|
||||
* are talking about the same games — otherwise two numbers that ought to agree would not, for a
|
||||
* reason nobody would find.
|
||||
*/
|
||||
function runOne(policy: BotPolicy, seed: number, length: GameLength, mode: GameMode): GameStats {
|
||||
const s = createGame({ id: `cmp-${seed}`, seed, config: SOLO(length, mode), playerNames: ['bot'] });
|
||||
const probe = makeFunnelProbe(0);
|
||||
const r = playGame(s, policy, pump, 50_000, probe.onEvent, probe.onTurn);
|
||||
return summarize(seed, r.events, r.intents, s, probe.funnel);
|
||||
}
|
||||
|
||||
export function compare(
|
||||
tweaks: BotTweaks,
|
||||
games: number,
|
||||
length: GameLength = 'standard',
|
||||
mode: GameMode = 'solitaire',
|
||||
): PairedResult {
|
||||
const variantPolicy = makeDeveloperBot(tweaks);
|
||||
const baseStats: GameStats[] = [];
|
||||
const variantStats: GameStats[] = [];
|
||||
const deltas: { seed: number; delta: number }[] = [];
|
||||
|
||||
for (let i = 0; i < games; i++) {
|
||||
const seed = 1000 + i * 7919; // the same prime stride the harness deals
|
||||
const a = runOne(developerBot, seed, length, mode);
|
||||
const b = runOne(variantPolicy, seed, length, mode);
|
||||
baseStats.push(a);
|
||||
variantStats.push(b);
|
||||
deltas.push({ seed, delta: b.revenue.net - a.revenue.net });
|
||||
}
|
||||
|
||||
const d = deltas.map((x) => x.delta);
|
||||
const mean = d.reduce((p, c) => p + c, 0) / Math.max(1, d.length);
|
||||
const sd =
|
||||
d.length > 1
|
||||
? Math.sqrt(d.reduce((p, c) => p + (c - mean) ** 2, 0) / (d.length - 1))
|
||||
: 0;
|
||||
const stderr = d.length > 0 ? sd / Math.sqrt(d.length) : 0;
|
||||
|
||||
return {
|
||||
seeds: games,
|
||||
baseline: developerBot.name,
|
||||
variant: variantPolicy.name,
|
||||
deltas,
|
||||
mean,
|
||||
sd,
|
||||
stderr,
|
||||
t: stderr === 0 ? 0 : mean / stderr,
|
||||
better: d.filter((x) => x > 0).length,
|
||||
worse: d.filter((x) => x < 0).length,
|
||||
identical: d.filter((x) => x === 0).length,
|
||||
baseStats,
|
||||
variantStats,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reporting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const mean = (xs: number[]): number => (xs.length ? xs.reduce((p, c) => p + c, 0) / xs.length : 0);
|
||||
|
||||
/**
|
||||
* The verdict, stated in the same terms every time.
|
||||
*
|
||||
* The threshold is t ≥ 3 rather than the conventional 2. This is a measurement taken repeatedly on
|
||||
* the same system while looking for something that works, so the conventional bar would have us keep
|
||||
* roughly one bad heuristic in twenty; and the cost of a wrong keep is not a wrong paper, it is a bot
|
||||
* that quietly plays worse and takes every later measurement with it.
|
||||
*/
|
||||
function verdict(t: number): string {
|
||||
const a = Math.abs(t);
|
||||
if (a >= 3) return t > 0 ? 'KEEP — clears the bar (t ≥ 3)' : 'REJECT — significantly worse';
|
||||
if (a >= 2) return 'NOT PROVEN — suggestive, run more seeds before believing it';
|
||||
return 'NO EFFECT MEASURED — inside the noise';
|
||||
}
|
||||
|
||||
export function formatPaired(r: PairedResult): string {
|
||||
const out: string[] = [];
|
||||
const num = (x: number, w = 8, dp = 2): string => x.toFixed(dp).padStart(w);
|
||||
|
||||
out.push(`\n=== paired comparison · ${r.seeds} seeds · same deal to both ===\n`);
|
||||
out.push(` baseline ${r.baseline}`);
|
||||
out.push(` variant ${r.variant}\n`);
|
||||
|
||||
const rows: [string, (g: GameStats) => number][] = [
|
||||
['revenue', (g) => g.revenue.net],
|
||||
['collisions', (g) => g.collisions],
|
||||
['trains scheduled', (g) => g.trains.scheduled],
|
||||
['arrivals', (g) => g.trains.arrived],
|
||||
['passengers on/off', (g) => g.revenue.passengerBoard + g.revenue.passengerDetrain],
|
||||
['freight loads+unloads', (g) => g.revenue.freightLoad + g.revenue.freightUnload],
|
||||
['cards played', (g) => g.development.cardsPlayed],
|
||||
['final grid size', (g) => g.development.gridSize],
|
||||
];
|
||||
out.push(' baseline variant delta');
|
||||
for (const [label, pick] of rows) {
|
||||
const a = mean(r.baseStats.map(pick));
|
||||
const b = mean(r.variantStats.map(pick));
|
||||
out.push(` ${label.padEnd(22)}${num(a)} ${num(b)} ${num(b - a)}`);
|
||||
}
|
||||
|
||||
out.push(
|
||||
`\n REVENUE DELTA ${r.mean >= 0 ? '+' : ''}${r.mean.toFixed(2)} ± ${r.stderr.toFixed(2)}` +
|
||||
` (t = ${r.t.toFixed(2)}, σ of the paired difference ${r.sd.toFixed(2)})`,
|
||||
);
|
||||
out.push(` ${verdict(r.t)}`);
|
||||
out.push(
|
||||
`\n seeds better ${r.better} · worse ${r.worse} · identical ${r.identical}` +
|
||||
` — ${((r.identical / Math.max(1, r.seeds)) * 100).toFixed(0)}% of games are untouched by this change`,
|
||||
);
|
||||
|
||||
// The tails, BY SEED, so a loss can be replayed rather than averaged away.
|
||||
const sorted = [...r.deltas].sort((a, b) => a.delta - b.delta);
|
||||
const show = (xs: { seed: number; delta: number }[]): string =>
|
||||
xs.map((x) => `${x.seed} (${x.delta > 0 ? '+' : ''}${x.delta})`).join(', ');
|
||||
out.push(` worst seeds: ${show(sorted.slice(0, 3))}`);
|
||||
out.push(` best seeds: ${show(sorted.slice(-3).reverse())}`);
|
||||
|
||||
// A change that raises revenue by abandoning a channel has to be visible as that.
|
||||
out.push('\n --- baseline funnel ---');
|
||||
out.push(funnelReport(r.baseStats));
|
||||
out.push('\n --- variant funnel ---');
|
||||
out.push(funnelReport(r.variantStats));
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `trainCapSlack=1` -> `{ trainCapSlack: 1 }`. Unknown names are refused rather than ignored. */
|
||||
export function parseTweaks(args: string[]): BotTweaks {
|
||||
const known = new Set(['trainCapSlack']);
|
||||
const tweaks: Record<string, number> = {};
|
||||
for (const a of args) {
|
||||
const m = /^([A-Za-z]\w*)=(-?\d+(?:\.\d+)?)$/.exec(a);
|
||||
if (!m) continue;
|
||||
if (!known.has(m[1]!)) {
|
||||
throw new Error(`unknown tweak "${m[1]}" — known: ${[...known].join(', ')}`);
|
||||
}
|
||||
tweaks[m[1]!] = Number(m[2]);
|
||||
}
|
||||
return tweaks as BotTweaks;
|
||||
}
|
||||
|
||||
const isMain = process.argv[1]?.endsWith('compare.ts') ?? false;
|
||||
|
||||
if (isMain) {
|
||||
const args = process.argv.slice(2);
|
||||
const games = Number(args.find((a) => /^\d+$/.test(a)) ?? 400);
|
||||
const lengthIdx = args.indexOf('--length');
|
||||
const length = (lengthIdx >= 0 ? args[lengthIdx + 1] : 'standard') as GameLength;
|
||||
const tweaks = parseTweaks(args);
|
||||
|
||||
if (Object.keys(tweaks).length === 0) {
|
||||
console.error('nothing to compare — pass at least one tweak, e.g. trainCapSlack=1');
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
console.log(formatPaired(compare(tweaks, games, length)));
|
||||
}
|
||||
}
|
||||
|
||||
export type { Funnel };
|
||||
+6
-3
@@ -19,7 +19,7 @@ import type { GameConfig, GameMode } from '../engine/state.ts';
|
||||
import type { BotPolicy, PlayOutcome } from './bot.ts';
|
||||
import { developerBot, playGame, randomBot } from './bot.ts';
|
||||
import type { GameStats } from './stats.ts';
|
||||
import { formatAggregate, summarize } from './stats.ts';
|
||||
import { formatAggregate, makeFunnelProbe, summarize } from './stats.ts';
|
||||
|
||||
export type SimOptions = {
|
||||
games: number;
|
||||
@@ -86,9 +86,12 @@ export function simulate(opts: SimOptions): SimReport {
|
||||
config: configFor(opts.mode, opts.length),
|
||||
playerNames: opts.players,
|
||||
});
|
||||
const r = playGame(s, opts.policy, pump);
|
||||
// The probe watches the game as it is played: the funnel gates are conditions at a moment, not
|
||||
// events, so they cannot be recovered from the log afterwards.
|
||||
const probe = makeFunnelProbe(0);
|
||||
const r = playGame(s, opts.policy, pump, 50_000, probe.onEvent, probe.onTurn);
|
||||
results.push(r);
|
||||
stats.push(summarize(seed, r.events, r.intents, s));
|
||||
stats.push(summarize(seed, r.events, r.intents, s, probe.funnel));
|
||||
}
|
||||
|
||||
const finished = results.filter((r) => r.finished);
|
||||
|
||||
@@ -18,11 +18,149 @@
|
||||
import type { GameEvent } from '../engine/events.ts';
|
||||
import type { Intent } from '../engine/intents.ts';
|
||||
import type { GameState } from '../engine/state.ts';
|
||||
import { areaOf, facilityCarTypes } from '../engine/apply.ts';
|
||||
import { officeProfile } from '../engine/content.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-game summary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* THE FUNNEL — how far each opportunity got before it stopped.
|
||||
*
|
||||
* The totals above say what happened; this says what ALMOST happened, which is what a change to the
|
||||
* bot has to move. Revenue can rise because a channel started working or because a channel was
|
||||
* abandoned in favour of a cheaper one, and the two look identical in a single number: capping the
|
||||
* trains a bot schedules raises revenue while cutting freight events nearly in half.
|
||||
*
|
||||
* SAMPLED FROM LIVE STATE, not from events, because the interesting gates are conditions rather than
|
||||
* occurrences — "was a green box stocked while a car was spotted" is not a thing that happens, it is
|
||||
* a thing that is true or false at a moment. `playGame` calls the probe once per decision and the
|
||||
* probe dedupes per phase, so these count PHASES, not turns.
|
||||
*/
|
||||
export type Funnel = {
|
||||
/** Trains reaching an Office, and what they were carrying when they got there. */
|
||||
arrivals: number;
|
||||
arrivalsAtPassengerOffice: number;
|
||||
/** Boarding needs an empty coach aboard (§9.2); detraining needs a loaded one. */
|
||||
arrivalsWithEmptyCoach: number;
|
||||
arrivalsWithLoadedCoach: number;
|
||||
/** A car of a type some industry in the district actually wants, loaded or empty. */
|
||||
arrivalsWithWantedCar: number;
|
||||
|
||||
/** Cargo phases, and how many had each half of what a load needs. */
|
||||
cargoPhases: number;
|
||||
cargoWithFacility: number;
|
||||
cargoGreenStocked: number;
|
||||
cargoCarSpotted: number;
|
||||
cargoLaborerFree: number;
|
||||
/** All three at ONE industry — the only combination that can actually move a load. */
|
||||
cargoReady: number;
|
||||
|
||||
/**
|
||||
* Decisions taken with a train's engine buried mid-consist, and how many of those had a legal way
|
||||
* to dig it out. §8.2 will not let such a train leave, and Rolling Stock may not be set out at the
|
||||
* Office — so the gap between these two numbers is the trap the bot cannot escape from.
|
||||
*/
|
||||
buriedTurns: number;
|
||||
buriedWithDigAvailable: number;
|
||||
};
|
||||
|
||||
function emptyFunnel(): Funnel {
|
||||
return {
|
||||
arrivals: 0, arrivalsAtPassengerOffice: 0, arrivalsWithEmptyCoach: 0,
|
||||
arrivalsWithLoadedCoach: 0, arrivalsWithWantedCar: 0,
|
||||
cargoPhases: 0, cargoWithFacility: 0, cargoGreenStocked: 0, cargoCarSpotted: 0,
|
||||
cargoLaborerFree: 0, cargoReady: 0,
|
||||
buriedTurns: 0, buriedWithDigAvailable: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A probe to hand to `playGame`: an event hook, a per-decision hook, and the totals they fill.
|
||||
*
|
||||
* One per game. Kept here beside the statistics it produces rather than in the harness, so the
|
||||
* definition of "a Cargo phase that was ready" lives in exactly one place.
|
||||
*/
|
||||
export function makeFunnelProbe(player = 0): {
|
||||
funnel: Funnel;
|
||||
onEvent: (e: GameEvent, s: GameState) => void;
|
||||
onTurn: (s: GameState) => void;
|
||||
} {
|
||||
const funnel = emptyFunnel();
|
||||
// Phases are sampled once, on the first decision taken in them.
|
||||
let lastPhaseKey = '';
|
||||
|
||||
const onEvent = (e: GameEvent, s: GameState): void => {
|
||||
if (e.type !== 'trainArrived') return;
|
||||
funnel.arrivals += 1;
|
||||
const area = areaOf(s, player);
|
||||
if (officeProfile(area.tier).isPassengerFacility) funnel.arrivalsAtPassengerOffice += 1;
|
||||
if (e.consist.some((c) => c.type === 'coach' && !c.loaded)) funnel.arrivalsWithEmptyCoach += 1;
|
||||
if (e.consist.some((c) => c.type === 'coach' && c.loaded)) funnel.arrivalsWithLoadedCoach += 1;
|
||||
|
||||
for (const card of area.grid.values()) {
|
||||
const f = card.facility;
|
||||
if (!f || f.kind !== 'freight') continue;
|
||||
const want = facilityCarTypes(f);
|
||||
const useful = e.consist.some(
|
||||
(c) =>
|
||||
want.includes(c.type) &&
|
||||
((f.allows.outbound && !c.loaded) || (f.allows.inbound && c.loaded)),
|
||||
);
|
||||
if (useful) {
|
||||
funnel.arrivalsWithWantedCar += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const onTurn = (s: GameState): void => {
|
||||
const area = areaOf(s, player);
|
||||
|
||||
// A buried engine is a per-decision condition: every turn it persists is a turn the train is
|
||||
// stuck, so this counts turns rather than trains.
|
||||
for (const tray of s.trays.values()) {
|
||||
if (tray.position.at !== 'grid' || tray.position.owner !== player) continue;
|
||||
if (tray.engineAt <= 0 || tray.engineAt >= tray.consist.length) continue;
|
||||
funnel.buriedTurns += 1;
|
||||
// Cars may never be set out at the Office (§A.4), which is where this almost always happens.
|
||||
const atOffice =
|
||||
tray.position.coord.row === area.officeCoord.row &&
|
||||
tray.position.coord.col === area.officeCoord.col;
|
||||
if (!atOffice) funnel.buriedWithDigAvailable += 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (s.clock.phase !== 'loadUnload') return;
|
||||
const key = `${s.clock.day}/${s.clock.stage}`;
|
||||
if (key === lastPhaseKey) return;
|
||||
lastPhaseKey = key;
|
||||
|
||||
funnel.cargoPhases += 1;
|
||||
let facility = false, green = false, car = false, laborer = false, ready = false;
|
||||
for (const card of area.grid.values()) {
|
||||
const f = card.facility;
|
||||
if (!f || f.kind !== 'freight') continue;
|
||||
facility = true;
|
||||
const g = f.outboundBox.length > 0;
|
||||
const c = f.industryTrack.cars.length > 0;
|
||||
const l = f.laborers - f.usedThisStage.laborers > 0;
|
||||
green ||= g;
|
||||
car ||= c;
|
||||
laborer ||= l;
|
||||
ready ||= g && c && l;
|
||||
}
|
||||
if (facility) funnel.cargoWithFacility += 1;
|
||||
if (green) funnel.cargoGreenStocked += 1;
|
||||
if (car) funnel.cargoCarSpotted += 1;
|
||||
if (laborer) funnel.cargoLaborerFree += 1;
|
||||
if (ready) funnel.cargoReady += 1;
|
||||
};
|
||||
|
||||
return { funnel, onEvent, onTurn };
|
||||
}
|
||||
|
||||
export type RevenueBreakdown = {
|
||||
freightLoad: number;
|
||||
freightUnload: number;
|
||||
@@ -76,6 +214,11 @@ export type GameStats = {
|
||||
collisions: number;
|
||||
eventCounts: Record<string, number>;
|
||||
intentCounts: Record<string, number>;
|
||||
/**
|
||||
* How far each opportunity got. Optional because it needs a live probe during the game, which a
|
||||
* caller reconstructing statistics from a saved event log cannot supply.
|
||||
*/
|
||||
funnel?: Funnel;
|
||||
};
|
||||
|
||||
function inc(map: Record<string, number>, key: string, by = 1): void {
|
||||
@@ -87,6 +230,7 @@ export function summarize(
|
||||
events: GameEvent[],
|
||||
intents: Intent['type'][],
|
||||
final: GameState,
|
||||
funnel?: Funnel,
|
||||
): GameStats {
|
||||
const eventCounts: Record<string, number> = {};
|
||||
const intentCounts: Record<string, number> = {};
|
||||
@@ -211,6 +355,7 @@ export function summarize(
|
||||
|
||||
eventCounts,
|
||||
intentCounts,
|
||||
...(funnel ? { funnel } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -414,6 +559,44 @@ export function formatGameStats(g: GameStats): string {
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* THE FUNNEL, AS PERCENTAGES OF THE THING THEY GATE.
|
||||
*
|
||||
* Deliberately not per-game averages: "3.6 arrivals carried an empty coach" says nothing without the
|
||||
* 7.3 arrivals it is out of, and the whole point is to see WHERE an opportunity stops. Read the
|
||||
* arrivals block as "of every train that reached an Office" and the Cargo block as "of every Cargo
|
||||
* phase". Empty when nothing was probed.
|
||||
*/
|
||||
export function funnelReport(all: GameStats[]): string {
|
||||
const fs = all.map((g) => g.funnel).filter((f): f is Funnel => f !== undefined);
|
||||
if (fs.length === 0) return '';
|
||||
const sum = (pick: (f: Funnel) => number): number => fs.reduce((n, f) => n + pick(f), 0);
|
||||
const pct = (n: number, d: number): string =>
|
||||
d === 0 ? ' —' : `${((n / d) * 100).toFixed(0).padStart(3)}%`;
|
||||
|
||||
const arr = sum((f) => f.arrivals);
|
||||
const cargo = sum((f) => f.cargoPhases);
|
||||
const out: string[] = [];
|
||||
|
||||
out.push(`\n PASSENGER FUNNEL — ${(arr / fs.length).toFixed(1)} arrivals a game`);
|
||||
out.push(` at a Passenger Facility (Depot+) ${pct(sum((f) => f.arrivalsAtPassengerOffice), arr)}`);
|
||||
out.push(` carrying an EMPTY coach (boarding) ${pct(sum((f) => f.arrivalsWithEmptyCoach), arr)}`);
|
||||
out.push(` carrying a LOADED coach (detrain) ${pct(sum((f) => f.arrivalsWithLoadedCoach), arr)}`);
|
||||
out.push(` carrying a car an industry wants ${pct(sum((f) => f.arrivalsWithWantedCar), arr)}`);
|
||||
|
||||
out.push(`\n FREIGHT FUNNEL — ${(cargo / fs.length).toFixed(0)} Cargo phases a game`);
|
||||
out.push(` a freight facility was built ${pct(sum((f) => f.cargoWithFacility), cargo)}`);
|
||||
out.push(` a green box was stocked ${pct(sum((f) => f.cargoGreenStocked), cargo)}`);
|
||||
out.push(` a car was spotted on an industry ${pct(sum((f) => f.cargoCarSpotted), cargo)}`);
|
||||
out.push(` a Laborer was free ${pct(sum((f) => f.cargoLaborerFree), cargo)}`);
|
||||
out.push(` ALL THREE at one industry ${pct(sum((f) => f.cargoReady), cargo)}`);
|
||||
|
||||
const buried = sum((f) => f.buriedTurns);
|
||||
out.push(`\n STUCK — ${(buried / fs.length).toFixed(1)} decisions a game with the engine buried mid-train`);
|
||||
out.push(` ... of those, able to set the nose cars out ${pct(sum((f) => f.buriedWithDigAvailable), buried)}`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function formatAggregate(all: GameStats[]): string {
|
||||
const out: string[] = [];
|
||||
out.push(`\n=== end-of-game statistics · ${all.length} games ===\n`);
|
||||
@@ -449,6 +632,8 @@ export function formatAggregate(all: GameStats[]): string {
|
||||
out.push(` ${t.padEnd(14)} ${n} (${((n / all.length) * 100).toFixed(0)}%)`);
|
||||
}
|
||||
|
||||
out.push(funnelReport(all));
|
||||
|
||||
out.push('\n ACTION MIX (mean per game)');
|
||||
out.push(` switch moves ${num(meanOf(all, (g) => g.actions.moves))}`);
|
||||
out.push(` cars dropped ${num(meanOf(all, (g) => g.actions.drops))}`);
|
||||
|
||||
Reference in New Issue
Block a user