portal/tools/brain-retro-analyzer.mjs

#!/usr/bin/env node
/**
 * Brain-retro analyzer (brain governance, observer factor-analysis spec §6).
 * Pure, deterministic Layer-4 aggregation over observer episodes for the
 * /brain-retro skill. Read-only — never writes JSONL. No LLM.
 *
 * Security Guidance #40: pure parsing — no exec/execSync.
 */
import { readFileSync, existsSync } from 'fs';

const SIZE_SMALL = 20;
const SIZE_LARGE = 60;

/**
 * Deduplicate the routing-gate double-write: a turn that was blocked then
 * re-stopped yields two episodes with the same task_id + started_at. Keep the
 * last (most complete). observer_error markers are all kept.
 */
export function dedupeEpisodes(episodes) {
  const errors = episodes.filter((e) => e && e.observer_error);
  const normal = episodes.filter((e) => e && !e.observer_error);
  const byKey = new Map();
  for (const e of normal) {
    byKey.set(`${e.task_id}|${(e.timestamps || {}).started_at}`, e);
  }
  return [...byKey.values(), ...errors];
}

/** Infer the true outcome of an episode from its events + the next episode's prompt. */
export function inferOutcome(episode, nextEpisode) {
  const events = episode && Array.isArray(episode.events) ? episode.events : [];
  if (events.some((e) => e.kind === 'interrupt')) {
    return 'partial';
  }
  // A turn that hit more tool errors than it retried away ended on an
  // unrecovered failure — the work was blocked, not merely reworked later.
  const errorCount = events.filter((e) => e.kind === 'error').length;
  const retryCount = events.filter((e) => e.kind === 'retry').length;
  if (errorCount > retryCount) {
    return 'blocked';
  }
  // 'failure' (work wrong AND never corrected) is a judgment, not
  // deterministically recoverable from a transcript — deferred to the phase-2
  // agent-judge. Until then a wrong-then-corrected turn surfaces as 'rework'.
  if (!nextEpisode) return 'unknown';
  if (nextEpisode.prompt_signal === 'correction') return 'rework';
  if (nextEpisode.prompt_signal === 'approval' || nextEpisode.prompt_signal === 'new_task') return 'success';
  return 'unknown';
}

function bySessionSorted(episodes) {
  const map = new Map();
  for (const e of episodes) {
    if (e.observer_error) continue;
    const sid = e.task_id || 'unknown';
    if (!map.has(sid)) map.set(sid, []);
    map.get(sid).push(e);
  }
  for (const eps of map.values()) {
    eps.sort((a, b) =>
      String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at))
    );
  }
  return map;
}

/** Group episodes into tasks: a new task starts after a success or on a new_task prompt. */
export function groupEpisodesToTasks(episodes) {
  const tasks = [];
  for (const [sid, eps] of bySessionSorted(episodes)) {
    let current = null;
    eps.forEach((episode, i) => {
      const prev = eps[i - 1];
      const prevOutcome = prev ? inferOutcome(prev, episode) : null;
      const isNewTask = i === 0 || prevOutcome === 'success' || episode.prompt_signal === 'new_task';
      if (isNewTask) {
        current = { task_ref: `${sid}#${tasks.length + 1}`, episodes: [] };
        tasks.push(current);
      }
      current.episodes.push(episode);
    });
  }
  return tasks;
}

/** Causal-chain candidates: an errored episode → a later episode sharing a file. */
export function findCausalChains(episodes) {
  const sorted = episodes
    .filter((e) => !e.observer_error)
    .slice()
    .sort((a, b) =>
      String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at))
    );
  const chains = [];
  for (let i = 0; i < sorted.length - 1; i++) {
    const a = sorted[i];
    const hasError = Array.isArray(a.events) && a.events.some((e) => e.kind === 'error');
    if (!hasError) continue;
    const filesA = new Set(((a.task_size || {}).files) || []);
    if (filesA.size === 0) continue;
    for (let j = i + 1; j < sorted.length; j++) {
      const b = sorted[j];
      const shared = (((b.task_size || {}).files) || []).filter((f) => filesA.has(f));
      if (shared.length > 0) {
        chains.push({
          from: `${a.task_id}|${(a.timestamps || {}).started_at}`,
          to: `${b.task_id}|${(b.timestamps || {}).started_at}`,
          sharedFiles: shared,
        });
        break;
      }
    }
  }
  return chains;
}

function sizeBucket(toolCalls) {
  const n = Number(toolCalls) || 0;
  return n < SIZE_SMALL ? 'small' : n <= SIZE_LARGE ? 'medium' : 'large';
}

const SESSION_TURN_EARLY = 10;
const SESSION_TURN_LATE = 40;

function sessionTurnBucket(turn) {
  const n = Number(turn);
  if (!Number.isFinite(n)) return 'null';
  return n < SESSION_TURN_EARLY ? 'early' : n <= SESSION_TURN_LATE ? 'mid' : 'late';
}

const FACTOR_FNS = {
  decision_provenance: (e) => (e.decision_provenance || {}).kind || 'unknown',
  economy_level: (e) => String((e.environment || {}).economy_level ?? 'null'),
  model: (e) => (e.environment || {}).model || 'null',
  post_compaction: (e) => String((e.environment || {}).post_compaction ?? false),
  session_turn: (e) => sessionTurnBucket((e.environment || {}).session_turn),
  parallel_session: (e) => String((e.environment || {}).parallel_session ?? false),
  task_size: (e) => sizeBucket((e.task_size || {}).tool_calls),
  node_chosen: (e) => (e.primary_rationale || {}).node_chosen || 'direct',
  task_classification: (e) => (e.primary_rationale || {}).task_classification || 'other',
};

/** Factor matrix: rows = factor values, columns = outcome distribution (spec §6). */
export function buildFactorMatrix(episodesWithOutcome) {
  const matrix = {};
  for (const [fname, fn] of Object.entries(FACTOR_FNS)) {
    matrix[fname] = {};
    for (const e of episodesWithOutcome) {
      const val = fn(e);
      const outcome = e._inferredOutcome || 'unknown';
      matrix[fname][val] = matrix[fname][val] || {};
      matrix[fname][val][outcome] = (matrix[fname][val][outcome] || 0) + 1;
    }
  }
  return matrix;
}

/** Full deterministic aggregation: dedup → infer outcomes → group → chains → matrix. */
export function analyze(episodes) {
  const deduped = dedupeEpisodes(episodes);
  const allNormal = deduped.filter((e) => !e.observer_error);
  // v1 episodes lack environment / prompt_signal / decision_provenance — they
  // pollute the factor matrix and break outcome inference. Analyze v2 only.
  const normal = allNormal.filter((e) => e.schema_version === 2);
  const v1SkippedCount = allNormal.length - normal.length;
  for (const eps of bySessionSorted(normal).values()) {
    eps.forEach((episode, i) => {
      episode._inferredOutcome = inferOutcome(episode, eps[i + 1]);
    });
  }
  return {
    episodeCount: normal.length,
    v1SkippedCount,
    observerErrorCount: deduped.length - allNormal.length,
    tasks: groupEpisodesToTasks(normal),
    causalChains: findCausalChains(normal),
    factorMatrix: buildFactorMatrix(normal),
  };
}

function loadEpisodes(files) {
  const eps = [];
  for (const f of files) {
    if (!existsSync(f)) continue;
    for (const line of readFileSync(f, 'utf-8').split('\n')) {
      const t = line.trim();
      if (!t) continue;
      try {
        eps.push(JSON.parse(t));
      } catch {
        // skip broken line
      }
    }
  }
  return eps;
}

if (process.argv[1] && process.argv[1].replace(/\\/g, '/').endsWith('/brain-retro-analyzer.mjs')) {
  const result = analyze(loadEpisodes(process.argv.slice(2)));
  console.log(JSON.stringify(result, null, 2));
  process.exit(0);
}