TheLadder/tools/analyze-log.js

// Read an exported feedback log and report what actually happened.
//
//   node tools/analyze-log.js theladder-feedback-2026-09-10-11-04-22.json
//
// The question this exists to answer: how many turns were played against how
// many distinct situations, and how often each one came round again. A pack
// that invites a 180-turn run needs to be judged on what turn 180 looked like,
// not on what turn 5 did.
//
// It also surfaces the two things worth more than any rating — the gates
// players kept bouncing off, and anything they said they wanted to do.

import { readFile, readdir, stat } from 'node:fs/promises';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const logsDir = resolve(root, 'logs');

/** The most recently modified log in logs/, when none was named. */
async function newestLog() {
  let names;
  try {
    names = (await readdir(logsDir)).filter((name) => name.endsWith('.json'));
  } catch {
    return null;
  }
  if (!names.length) return null;

  const timed = await Promise.all(names.map(async (name) => {
    const path = join(logsDir, name);
    return { path, at: (await stat(path)).mtimeMs };
  }));
  return timed.sort((a, b) => b.at - a.at)[0].path;
}

const bar = (fraction, width = 24) =>
  '█'.repeat(Math.max(1, Math.round(fraction * width))).padEnd(width, '·');

const pct = (n, total) => `${((n / total) * 100).toFixed(1)}%`;

function section(title) {
  console.log(`\n${title}`);
  console.log('─'.repeat(title.length));
}

/** Split the log into playthroughs. One export can hold several. */
function byRun(entries) {
  const runs = new Map();
  for (const entry of entries) {
    if (!runs.has(entry.seed)) runs.set(entry.seed, []);
    runs.get(entry.seed).push(entry);
  }
  return runs;
}

function reportRepetition(entries) {
  const counts = new Map();
  const firstSeen = new Map();
  const lastSeen = new Map();
  for (const entry of entries) {
    counts.set(entry.event_id, (counts.get(entry.event_id) ?? 0) + 1);
    if (!firstSeen.has(entry.event_id)) firstSeen.set(entry.event_id, entry.turn);
    lastSeen.set(entry.event_id, entry.turn);
  }

  const distinct = counts.size;
  const turns = entries.length;
  section('Repetition');
  console.log(`${turns} turns across ${distinct} distinct situations ` +
    `— each seen ${(turns / distinct).toFixed(1)} times on average.`);
  const repeated = [...counts.values()].filter((n) => n > 1).length;
  console.log(`${repeated} of ${distinct} came up more than once; ` +
    `${distinct - repeated} were one-offs.`);

  const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
  const most = ranked[0][1];
  console.log('');
  for (const [eventId, count] of ranked) {
    const gap = count > 1
      ? ` every ~${Math.round((lastSeen.get(eventId) - firstSeen.get(eventId)) / (count - 1))} turns`
      : ` once, turn ${firstSeen.get(eventId)}`;
    console.log(`  ${eventId.padEnd(24)} ${String(count).padStart(3)}  ` +
      `${bar(count / most)} ${pct(count, turns).padStart(6)}${gap}`);
  }
}

function reportChoices(entries) {
  const counts = new Map();
  for (const entry of entries) {
    const key = `${entry.event_id}.${entry.action_taken}`;
    counts.set(key, (counts.get(key) ?? 0) + 1);
  }
  section('Choices');
  const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
  console.log(`${counts.size} distinct options taken. Most used:`);
  for (const [key, count] of ranked.slice(0, 8)) {
    console.log(`  ${String(count).padStart(3)}×  ${key}`);
  }
}

function reportGates(entries) {
  // What players were shown but could not take. The strongest signal that a
  // choice set is too narrow, and it arrives whether or not anyone types.
  const blocked = new Map();
  for (const entry of entries) {
    for (const option of entry.offered ?? []) {
      if (option.available) continue;
      const reasons = (option.failed ?? [])
        .map((c) => `${c.path} ${c.op} ${JSON.stringify(c.value)}`)
        .join(', ') || 'unstated';
      const key = `${entry.event_id}.${option.id} — needs ${reasons}`;
      blocked.set(key, (blocked.get(key) ?? 0) + 1);
    }
  }
  section('Doors that stayed shut');
  if (blocked.size === 0) {
    console.log('Nothing was ever locked.');
    return;
  }
  const ranked = [...blocked.entries()].sort((a, b) => b[1] - a[1]);
  console.log(`${ranked.reduce((s, [, n]) => s + n, 0)} locked options shown, ` +
    `${blocked.size} distinct. Most often:`);
  for (const [key, count] of ranked.slice(0, 10)) {
    console.log(`  ${String(count).padStart(3)}×  ${key}`);
  }
}

function reportFeedback(entries) {
  section('What the player said');
  const rated = entries.filter((e) => e.rating);
  const tally = { up: 0, neutral: 0, down: 0 };
  for (const entry of rated) tally[entry.rating] = (tally[entry.rating] ?? 0) + 1;

  console.log(rated.length
    ? `${rated.length} of ${entries.length} turns rated — ` +
      `👍 ${tally.up}  😐 ${tally.neutral}  👎 ${tally.down}`
    : 'No turns were rated.');

  const rateFor = (id) => {
    const seen = entries.filter((e) => e.event_id === id && e.rating);
    if (!seen.length) return null;
    const score = seen.reduce((s, e) => s + (e.rating === 'up' ? 1 : e.rating === 'down' ? -1 : 0), 0);
    return { id, score, n: seen.length };
  };
  const scored = [...new Set(entries.map((e) => e.event_id))]
    .map(rateFor).filter(Boolean).sort((a, b) => b.score - a.score);
  if (scored.length > 1) {
    console.log(`\n  best received: ${scored[0].id} (${scored[0].score >= 0 ? '+' : ''}${scored[0].score} over ${scored[0].n})`);
    const worst = scored.at(-1);
    console.log(`  worst received: ${worst.id} (${worst.score >= 0 ? '+' : ''}${worst.score} over ${worst.n})`);
  }

  const comments = entries.filter((e) => e.comment);
  if (comments.length) {
    console.log('\n  Comments:');
    for (const entry of comments) {
      console.log(`    turn ${entry.turn} (${entry.event_id}): ${entry.comment}`);
    }
  }

  const wanted = entries.filter((e) => e.desired_but_unavailable_action);
  if (wanted.length) {
    console.log('\n  Wanted to do, but could not:');
    for (const entry of wanted) {
      console.log(`    turn ${entry.turn} (${entry.event_id}): ${entry.desired_but_unavailable_action}`);
    }
  }
}

async function main() {
  const path = process.argv[2] ?? await newestLog();
  if (!path) {
    console.error('usage: npm run analyze -- <exported-feedback-log.json>');
    console.error('   or: drop the export into logs/ and run npm run analyze');
    process.exitCode = 1;
    return;
  }
  if (!process.argv[2]) console.log(`reading ${path.replace(`${root}/`, '')}\n`);

  const log = JSON.parse(await readFile(path, 'utf8'));
  const entries = log.entries ?? [];
  if (!entries.length) {
    console.log('The log has no entries.');
    return;
  }

  const runs = byRun(entries);
  console.log(`${entries.length} turns logged across ${runs.size} run${runs.size === 1 ? '' : 's'} ` +
    `(pack ${entries[0].pack_id} ${entries[0].pack_version ?? ''})`.trimEnd());
  for (const [seed, run] of runs) {
    const stages = [...new Set(run.map((e) => e.stage))];
    console.log(`  seed ${seed}: ${run.length} turns, ${stages.join(' → ')}`);
  }

  reportRepetition(entries);
  reportChoices(entries);
  reportGates(entries);
  reportFeedback(entries);
  console.log('');
}

main().catch((error) => {
  console.error(`could not analyse that log: ${error.message}`);
  process.exitCode = 1;
});