#!/usr/bin/env node
// AI Pricing Guru Labs — reasoning-effort experiment.
//
// Question: what does the same model cost per task at reasoning effort
// none/low/medium/high, and what accuracy does the extra spend buy?
// Per-token list prices do not change with effort; the number of (hidden)
// reasoning tokens does, and those are billed as output tokens.
//
// Sister runner of run-cost-per-task.mjs: same spend ledger, $100 monthly cap,
// archive layout, control-plane credit blocker and notify gate.
//
//   node /opt/aipricing/scripts/labs/run-reasoning-effort.mjs --adapter mock
//   node /opt/aipricing/scripts/labs/run-reasoning-effort.mjs --adapter openrouter --only gpt-6-luna,gemini-3.6-flash --budget 2
//   node /opt/aipricing/scripts/labs/run-reasoning-effort.mjs --adapter openrouter --budget 15
//
// Output: /opt/aipricing/data/labs/reasoning-effort-latest.json (+ archive copy).
// --public-output <path> also writes the same JSON elsewhere (e.g. the repo's
// public/labs/reasoning-effort.json); nothing is committed by this script.
import { notify as notifyGate } from "../lib/notify.mjs";
import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync, copyFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { performance } from "node:perf_hooks";
import { openStore } from "../control-plane/store.mjs";

const ROOT = "/opt/aipricing";
const SCRIPT_DIR = `${ROOT}/scripts/labs`;
const DATA_DIR = `${ROOT}/data/labs`;
const ARCHIVE_ROOT = `${DATA_DIR}/archive`;
const PRICING_PATH = process.env.AIPRICING_PRICING_PATH || `${ROOT}/repo/public/pricing.json`;
const OPENROUTER_ENV = `${ROOT}/credentials/openrouter.env`;
const SPEND_LOG = `${DATA_DIR}/spend.jsonl`;
const LAST_RUN = `${DATA_DIR}/last-run.json`;
const MONTHLY_CAP_USD = 100;
const WARN_AT_USD = 80;
const EXPERIMENT = "reasoning-effort";
const EFFORT_ORDER = ["none", "minimal", "low", "medium", "high", "xhigh", "max"];
const REQUEST_TIMEOUT_MS = 300_000;
const RATE_LIMIT_RETRY_DELAYS_MS = [30_000, 60_000, 120_000];

const DEFAULT_SYSTEM = [
  "You are running a deterministic benchmark.",
  "Follow the user instruction exactly.",
  "Return only the requested answer, with no explanation unless explicitly requested."
].join(" ");

function parseArgs(argv) {
  const args = {
    adapter: null,
    budget: 15,
    suite: `${SCRIPT_DIR}/reasoning-effort-tasks.v1.json`,
    models: `${SCRIPT_DIR}/reasoning-effort-models.v1.json`,
    output: `${DATA_DIR}/reasoning-effort-latest.json`,
    publicOutput: null,
    only: null,
    efforts: null,
    taskCount: null,
    maxTokens: 16000,
    concurrency: 4,
    noTelegram: false
  };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--adapter") args.adapter = argv[++i];
    else if (a === "--budget") args.budget = Number(argv[++i]);
    else if (a === "--suite") args.suite = argv[++i];
    else if (a === "--models") args.models = argv[++i];
    else if (a === "--output") args.output = argv[++i];
    else if (a === "--public-output") args.publicOutput = argv[++i];
    else if (a === "--only") args.only = argv[++i].split(",").map((s) => s.trim()).filter(Boolean);
    else if (a === "--efforts") args.efforts = argv[++i].split(",").map((s) => s.trim()).filter(Boolean);
    else if (a === "--task-count") args.taskCount = Number(argv[++i]);
    else if (a === "--max-tokens") args.maxTokens = Number(argv[++i]);
    else if (a === "--concurrency") args.concurrency = Number(argv[++i]);
    else if (a === "--no-telegram") args.noTelegram = true;
    else if (a === "--replay") { args.replay = argv[++i]; args.adapter = "replay"; }
    else if (a === "--help") {
      console.log("Usage: run-reasoning-effort.mjs [--adapter mock|openrouter] [--budget 15] [--only pricingId,...] [--efforts low,high] [--task-count N] [--max-tokens 16000] [--concurrency 4] [--public-output path] [--no-telegram]");
      process.exit(0);
    } else throw new Error(`unknown arg: ${a}`);
  }
  args.adapter ||= loadEnv(OPENROUTER_ENV).OPENROUTER_API_KEY ? "openrouter" : "mock";
  if (!["mock", "openrouter", "replay"].includes(args.adapter)) throw new Error(`unsupported adapter: ${args.adapter}`);
  if (args.adapter === "replay") {
    // Re-grade / re-cost an archived run with no API calls (and no spend).
    if (!args.replay || !existsSync(join(args.replay, "raw-responses.jsonl"))) throw new Error("--replay needs an archive dir containing raw-responses.jsonl");
    const archivedModels = join(args.replay, "models.json");
    if (existsSync(archivedModels)) args.models = archivedModels;
  }
  if (!Number.isFinite(args.budget) || args.budget <= 0) throw new Error("--budget must be positive");
  if (!Number.isInteger(args.maxTokens) || args.maxTokens < 1024) throw new Error("--max-tokens must be an integer >= 1024");
  if (!Number.isInteger(args.concurrency) || args.concurrency < 1 || args.concurrency > 12) throw new Error("--concurrency must be 1..12");
  if (args.taskCount != null && (!Number.isInteger(args.taskCount) || args.taskCount <= 0)) throw new Error("--task-count must be a positive integer");
  return args;
}

function loadEnv(path) {
  if (!existsSync(path)) return {};
  const env = {};
  for (const line of readFileSync(path, "utf8").split(/\r?\n/)) {
    const t = line.trim();
    if (!t || t.startsWith("#")) continue;
    const idx = t.indexOf("=");
    if (idx === -1) continue;
    env[t.slice(0, idx).trim()] = t.slice(idx + 1).trim().replace(/^["']|["']$/g, "");
  }
  return env;
}

const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
function writeJson(path, value) {
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
}

// ---------- grading (kept identical to run-cost-per-task.mjs) ----------
function normalize(s) {
  return String(s ?? "").trim()
    .replace(/^```(?:json|javascript|js|python)?\s*/i, "").replace(/```$/i, "")
    .trim().replace(/\s+/g, " ");
}
const normalizeLoose = (s) => normalize(s).toLowerCase().replace(/[.$]/g, "").trim();
function extractJson(text) {
  const clean = normalize(text);
  try { return JSON.parse(clean); } catch {}
  const start = clean.indexOf("{");
  const end = clean.lastIndexOf("}");
  if (start !== -1 && end > start) return JSON.parse(clean.slice(start, end + 1));
  throw new Error("no parseable JSON object");
}
function coerceScalar(v) {
  if (typeof v !== "string") return v;
  const t = v.trim();
  if (/^(true|false)$/i.test(t)) return /^true$/i.test(t);
  if (/^-?\d+(?:\.\d+)?$/.test(t)) return Number(t);
  return t;
}
function valuesEqual(actual, expected) {
  actual = coerceScalar(actual);
  expected = coerceScalar(expected);
  if (typeof expected === "number") return Math.abs(Number(actual) - expected) <= 1e-9;
  if (typeof expected === "boolean") return Boolean(actual) === expected;
  return normalizeLoose(actual) === normalizeLoose(expected);
}
// Unlike run-cost-per-task.mjs, strip pure presentation (markdown emphasis,
// inline code ticks, a leading currency sign, thousands separators) so that
// accuracy-vs-effort measures the answer, not formatting habits that differ by
// effort level. Explanations instead of a bare answer still fail.
function stripFormatting(s) {
  return normalize(s)
    .replace(/^(\*\*|__|`)+|(\*\*|__|`)+$/g, "")
    .replace(/^\$\s*/, "")
    .replace(/(\d),(?=\d{3}\b)/g, "$1")
    .trim();
}
function grade(output, task) {
  const g = task.grader;
  const clean = g.type === "number" || g.type === "exact" ? stripFormatting(output) : normalize(output);
  try {
    if (g.type === "exact") return { correct: normalizeLoose(clean) === normalizeLoose(g.value), detail: `expected ${g.value}` };
    if (g.type === "number") {
      const m = clean.match(/^(-?\d+(?:\.\d+)?)%?$/);
      const actual = m ? Number(m[1]) : NaN;
      return { correct: Number.isFinite(actual) && Math.abs(actual - g.value) <= (g.tolerance ?? 0), actual, detail: `expected ${g.value}` };
    }
    if (g.type === "regex") return { correct: new RegExp(g.pattern, g.flags || "i").test(clean), detail: `pattern ${g.pattern}` };
    if (g.type === "contains_all") {
      const hay = clean.toLowerCase();
      const missing = g.values.filter((v) => !hay.includes(String(v).toLowerCase()));
      const words = clean.match(/[A-Za-z0-9]+(?:[-.'’][A-Za-z0-9]+)*/g) || [];
      const sentenceParts = clean.split(/[.!?]+(?:\s+|$)/).map((p) => p.trim()).filter(Boolean);
      const constraintFailures = [];
      if (g.wordCount != null && words.length !== g.wordCount) constraintFailures.push(`wordCount=${words.length}, expected=${g.wordCount}`);
      if (g.maxWords != null && words.length > g.maxWords) constraintFailures.push(`wordCount=${words.length}, max=${g.maxWords}`);
      if (g.sentenceCount != null && sentenceParts.length !== g.sentenceCount) constraintFailures.push(`sentenceCount=${sentenceParts.length}, expected=${g.sentenceCount}`);
      return { correct: missing.length === 0 && constraintFailures.length === 0, missing, constraintFailures };
    }
    if (g.type === "json_fields") {
      const obj = extractJson(clean);
      const misses = [];
      for (const [key, expected] of Object.entries(g.fields)) if (!valuesEqual(obj[key], expected)) misses.push({ key, expected, actual: obj[key] });
      return { correct: misses.length === 0, actual: obj, missing: misses };
    }
    return { correct: false, detail: `unknown grader ${g.type}` };
  } catch (e) {
    return { correct: false, detail: e.message };
  }
}

// ---------- spend ----------
const monthPrefix = (d = new Date()) => d.toISOString().slice(0, 7);
function readMonthlySpend(month = monthPrefix()) {
  if (!existsSync(SPEND_LOG)) return 0;
  let total = 0;
  for (const line of readFileSync(SPEND_LOG, "utf8").split(/\r?\n/)) {
    if (!line.trim()) continue;
    try {
      const row = JSON.parse(line);
      if (String(row.ts || "").startsWith(month) && row.adapter !== "mock" && row.billable !== false) total += Number(row.billedUsd ?? row.spendUsd ?? 0);
    } catch {}
  }
  return total;
}
function appendSpend(row) {
  mkdirSync(dirname(SPEND_LOG), { recursive: true });
  appendFileSync(SPEND_LOG, `${JSON.stringify(row)}\n`);
}

function readUsage(usage) {
  const promptTokens = Number(usage?.prompt_tokens ?? 0);
  const completionTokens = Number(usage?.completion_tokens ?? 0);
  const rawReasoning = usage?.completion_tokens_details?.reasoning_tokens;
  const reasoningReported = rawReasoning != null;
  // Some routes occasionally report reasoning_tokens > completion_tokens.
  // completion_tokens is what gets billed, so cap reasoning at it and flag it.
  const reportedReasoning = reasoningReported ? Number(rawReasoning) : 0;
  const reasoningTokens = Math.min(reportedReasoning, completionTokens);
  const cachedTokens = Number(usage?.prompt_tokens_details?.cached_tokens ?? 0);
  return {
    promptTokens,
    completionTokens, // includes reasoning tokens (OpenRouter convention)
    reasoningTokens,
    visibleOutputTokens: Math.max(0, completionTokens - reasoningTokens),
    reasoningReported,
    reasoningOverReported: reportedReasoning > completionTokens ? reportedReasoning : null,
    cachedTokens
  };
}
function listCost(u, price) {
  const inputPerM = Number(price?.pricing?.inputPerM ?? 0);
  const outputPerM = Number(price?.pricing?.outputPerM ?? 0);
  const inputUsd = (u.promptTokens / 1e6) * inputPerM;
  const reasoningUsd = (u.reasoningTokens / 1e6) * outputPerM;
  const visibleOutputUsd = (u.visibleOutputTokens / 1e6) * outputPerM;
  return { inputPerM, outputPerM, inputUsd, reasoningUsd, visibleOutputUsd, totalUsd: inputUsd + reasoningUsd + visibleOutputUsd };
}
const estimateTokens = (text) => Math.max(1, Math.ceil(String(text).length / 4));
function estimateMaxCost(task, price, maxTokens) {
  const p = estimateTokens(`${DEFAULT_SYSTEM}\n${task.prompt}`);
  return (p * Number(price.pricing.inputPerM) + maxTokens * Number(price.pricing.outputPerM)) / 1e6;
}

// ---------- adapters ----------
function mockAnswer(task) {
  const g = task.grader;
  if (g.type === "exact" || g.type === "number") return String(g.value);
  if (g.type === "contains_all") {
    const parts = [...g.values];
    while (g.wordCount && parts.join(" ").split(/\s+/).length < g.wordCount) parts.push("summary");
    return parts.join(" ");
  }
  if (g.type === "regex") return g.pattern.replace(/[\^$\\]|\\s\*/g, "").replace(/s\*/g, "");
  if (g.type === "json_fields") return JSON.stringify(g.fields);
  return "";
}
async function callMock(model, effort, task) {
  const content = mockAnswer(task);
  const factor = { none: 0, minimal: 50, low: 200, medium: 800, high: 2500, xhigh: 5000, max: 8000 }[effort] ?? 500;
  const hardBoost = task.difficulty === "hard" ? 3 : 1;
  const reasoning = factor * hardBoost;
  const prompt = estimateTokens(`${DEFAULT_SYSTEM}\n${task.prompt}`);
  return {
    content,
    finishReason: "stop",
    usage: { prompt_tokens: prompt, completion_tokens: estimateTokens(content) + reasoning, completion_tokens_details: { reasoning_tokens: reasoning } },
    raw: { adapter: "mock" },
    latencyMs: 0
  };
}

function loadReplay(dir) {
  const map = new Map();
  for (const line of readFileSync(join(dir, "raw-responses.jsonl"), "utf8").split(/\r?\n/)) {
    if (!line.trim()) continue;
    const row = JSON.parse(line);
    map.set(`${row.pricingId}|${row.effort}|${row.taskId}`, row);
  }
  return map;
}
async function callReplay(replayMap, model, effort, task) {
  const row = replayMap.get(`${model.pricingId}|${effort}|${task.id}`);
  if (!row) throw new Error("not present in replayed archive");
  if (row.status === "error") throw new Error(row.error);
  return { content: row.output ?? "", finishReason: row.finishReason ?? null, usage: row.raw?.usage || {}, raw: row.raw, latencyMs: row.latencyMs ?? null, archivedBilledUsd: Number(row.billedUsd || 0) };
}

const delay = (ms) => new Promise((r) => setTimeout(r, ms));
function reasoningParam(effort) {
  if (effort === "none") return { effort: "none" };
  return { effort, exclude: true };
}
async function callOpenRouterOnce(model, effort, task, apiKey, maxTokens) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
  const started = performance.now();
  try {
    const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
      method: "POST",
      headers: {
        authorization: `Bearer ${apiKey}`,
        "content-type": "application/json",
        "http-referer": "https://www.aipricing.guru/labs/",
        "x-title": "AI Pricing Guru Labs"
      },
      // No temperature: several reasoning endpoints reject or override it.
      body: JSON.stringify({
        model: model.openrouterId,
        max_tokens: maxTokens,
        reasoning: reasoningParam(effort),
        usage: { include: true },
        messages: [
          { role: "system", content: DEFAULT_SYSTEM },
          { role: "user", content: task.prompt }
        ]
      }),
      signal: controller.signal
    });
    const json = await res.json().catch(() => ({}));
    const latencyMs = Math.round(performance.now() - started);
    if (!res.ok || json?.error) {
      const err = new Error(json?.error?.message || json?.message || `HTTP ${res.status}`);
      err.status = json?.error?.code ?? res.status;
      err.raw = json;
      throw err;
    }
    const choice = json.choices?.[0] || {};
    return { content: choice.message?.content ?? "", finishReason: choice.finish_reason ?? choice.native_finish_reason ?? null, usage: json.usage || {}, raw: json, latencyMs };
  } finally {
    clearTimeout(timer);
  }
}
async function callOpenRouter(model, effort, task, apiKey, maxTokens) {
  for (let attempt = 0; ; attempt++) {
    try {
      return await callOpenRouterOnce(model, effort, task, apiKey, maxTokens);
    } catch (error) {
      const retryDelayMs = RATE_LIMIT_RETRY_DELAYS_MS[attempt];
      if (error.status !== 429 || retryDelayMs == null) throw error;
      console.warn(JSON.stringify({ status: "retrying_rate_limit", model: model.openrouterId, effort, taskId: task.id, attempt: attempt + 1, retryDelayMs }));
      await delay(retryDelayMs);
    }
  }
}

// ---------- suite ----------
function loadSuite(path, taskCount) {
  const suite = readJson(path);
  let easy = [];
  if (suite.easyFrom && suite.easyTaskIds?.length) {
    const base = readJson(join(dirname(path), suite.easyFrom));
    const byId = new Map(base.tasks.map((t) => [t.id, t]));
    easy = suite.easyTaskIds.map((id) => {
      const t = byId.get(id);
      if (!t) throw new Error(`easy task ${id} not found in ${suite.easyFrom}`);
      return { ...t, difficulty: "easy" };
    });
  }
  const hard = suite.tasks.map((t) => ({ ...t, difficulty: t.difficulty || "hard" }));
  let tasks = [...easy, ...hard];
  if (taskCount) {
    // keep both tiers represented when trimming
    const half = Math.ceil(taskCount / 2);
    tasks = [...easy.slice(0, half), ...hard.slice(0, taskCount - Math.min(half, easy.length))];
  }
  return { suite, tasks };
}

// ---------- summaries ----------
const sum = (xs) => xs.reduce((a, b) => a + b, 0);
const mean = (xs) => (xs.length ? sum(xs) / xs.length : null);
function quantile(xs, q) {
  if (!xs.length) return null;
  const s = [...xs].sort((a, b) => a - b);
  const pos = (s.length - 1) * q;
  const lo = Math.floor(pos);
  const hi = Math.ceil(pos);
  return s[lo] + (s[hi] - s[lo]) * (pos - lo);
}
function tierStats(rows) {
  const ok = rows.filter((r) => r.status === "ok");
  const correct = ok.filter((r) => r.correct).length;
  const costs = ok.map((r) => r.listUsd);
  const reasoning = ok.map((r) => r.usage.reasoningTokens);
  return {
    attempted: ok.length,
    correct,
    accuracy: ok.length ? correct / ok.length : null,
    avgListUsdPerTask: mean(costs),
    costPerCorrectUsd: correct ? sum(costs) / correct : null,
    avgReasoningTokens: mean(reasoning),
    medianReasoningTokens: quantile(reasoning, 0.5),
    p90ReasoningTokens: quantile(reasoning, 0.9),
    maxReasoningTokens: reasoning.length ? Math.max(...reasoning) : null
  };
}
function summarizeCells(results, models, prices) {
  const cells = [];
  for (const model of models) {
    for (const effort of model.efforts) {
      const rows = results.filter((r) => r.pricingId === model.pricingId && r.effort === effort);
      if (!rows.length) continue;
      const ok = rows.filter((r) => r.status === "ok");
      const all = tierStats(rows);
      cells.push({
        pricingId: model.pricingId,
        label: model.label,
        effort,
        inputPerM: prices.get(model.pricingId)?.pricing?.inputPerM ?? null,
        outputPerM: prices.get(model.pricingId)?.pricing?.outputPerM ?? null,
        ...all,
        errorCount: rows.filter((r) => r.status === "error").length,
        skippedForBudget: rows.filter((r) => r.status === "skipped_budget").length,
        truncatedCount: ok.filter((r) => r.finishReason === "length").length,
        reasoningReported: ok.length ? ok.every((r) => r.usage.reasoningReported) : null,
        avgPromptTokens: mean(ok.map((r) => r.usage.promptTokens)),
        avgVisibleOutputTokens: mean(ok.map((r) => r.usage.visibleOutputTokens)),
        reasoningShareOfOutputCost: (() => {
          const out = sum(ok.map((r) => r.costBreakdown.reasoningUsd + r.costBreakdown.visibleOutputUsd));
          return out > 0 ? sum(ok.map((r) => r.costBreakdown.reasoningUsd)) / out : null;
        })(),
        listUsdTotal: sum(ok.map((r) => r.listUsd)),
        billedUsd: sum(rows.map((r) => Number(r.billedUsd || 0))),
        avgLatencyMs: mean(ok.map((r) => r.latencyMs).filter((x) => x != null)),
        byDifficulty: {
          easy: tierStats(rows.filter((r) => r.difficulty === "easy")),
          hard: tierStats(rows.filter((r) => r.difficulty === "hard"))
        }
      });
    }
  }
  return cells;
}
function summarizeModels(cells, models) {
  return models.map((model) => {
    const own = cells
      .filter((c) => c.pricingId === model.pricingId && c.attempted > 0)
      .sort((a, b) => EFFORT_ORDER.indexOf(a.effort) - EFFORT_ORDER.indexOf(b.effort));
    const baseline = own[0] || null;
    return {
      pricingId: model.pricingId,
      label: model.label,
      inputPerM: own[0]?.inputPerM ?? null,
      outputPerM: own[0]?.outputPerM ?? null,
      baselineEffort: baseline?.effort ?? null,
      efforts: own.map((c) => ({
        effort: c.effort,
        accuracy: c.accuracy,
        hardAccuracy: c.byDifficulty.hard.accuracy,
        avgListUsdPerTask: c.avgListUsdPerTask,
        hardAvgListUsdPerTask: c.byDifficulty.hard.avgListUsdPerTask,
        avgReasoningTokens: c.avgReasoningTokens,
        costMultiplierVsBaseline: baseline?.avgListUsdPerTask ? c.avgListUsdPerTask / baseline.avgListUsdPerTask : null,
        hardCostMultiplierVsBaseline: baseline?.byDifficulty.hard.avgListUsdPerTask ? c.byDifficulty.hard.avgListUsdPerTask / baseline.byDifficulty.hard.avgListUsdPerTask : null,
        accuracyDeltaVsBaseline: baseline?.accuracy != null && c.accuracy != null ? c.accuracy - baseline.accuracy : null,
        truncatedCount: c.truncatedCount,
        errorCount: c.errorCount
      })),
      unavailableEfforts: model.efforts.filter((e) => !own.some((c) => c.effort === e))
    };
  });
}

const fmt = (n, d = 4) => (n == null ? "n/a" : `$${Number(n).toFixed(d)}`);
function telegramText(result) {
  const lines = [
    "AI Pricing Guru Labs: reasoning-effort run complete.",
    `Run: ${result.runId}`,
    `Calls: ${result.attempted} ok, ${result.errors} errors, ${result.skippedForBudget} skipped; billed ${fmt(result.billedUsd)}.`
  ];
  for (const m of result.modelSummaries) {
    const parts = m.efforts.map((e) => `${e.effort} ${fmt(e.avgListUsdPerTask, 5)}/task ${e.costMultiplierVsBaseline == null ? "" : `(${e.costMultiplierVsBaseline.toFixed(1)}x)`} acc ${e.accuracy == null ? "n/a" : Math.round(e.accuracy * 100) + "%"}`);
    lines.push(`${m.label}: ${parts.join(" | ")}`);
  }
  lines.push(`Data: ${result.outputPath}`);
  return lines.join("\n");
}

async function main() {
  const args = parseArgs(process.argv.slice(2));

  if (args.adapter === "openrouter") {
    const control = openStore();
    const labsBlocker = control.getJob("labs:openrouter-credit");
    control.close();
    if (labsBlocker?.status === "blocked") {
      console.log(JSON.stringify({ status: "blocked", reason: labsBlocker.blockedReason, resumeCondition: labsBlocker.resumeCondition, controlPlaneJob: labsBlocker.id }));
      return;
    }
  }

  const { suite, tasks } = loadSuite(args.suite, args.taskCount);
  const modelSuite = readJson(args.models);
  let models = modelSuite.models;
  if (args.only) models = models.filter((m) => args.only.includes(m.pricingId) || args.only.includes(m.openrouterId));
  if (args.efforts) models = models.map((m) => ({ ...m, efforts: m.efforts.filter((e) => args.efforts.includes(e)) })).filter((m) => m.efforts.length);
  if (!models.length) throw new Error("no models selected");

  const pricing = readJson(PRICING_PATH);
  const prices = new Map(pricing.models.map((m) => [m.id, m]));
  for (const m of models) {
    const p = prices.get(m.pricingId);
    if (!p?.pricing?.inputPerM && p?.pricing?.inputPerM !== 0) throw new Error(`No list price in pricing.json for ${m.pricingId}`);
  }
  const env = loadEnv(OPENROUTER_ENV);
  if (args.adapter === "openrouter" && !env.OPENROUTER_API_KEY) throw new Error(`OPENROUTER_API_KEY missing in ${OPENROUTER_ENV}`);

  mkdirSync(ARCHIVE_ROOT, { recursive: true });
  const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${EXPERIMENT}-${args.adapter}`;
  const archiveDir = join(ARCHIVE_ROOT, runId);
  mkdirSync(archiveDir, { recursive: true });
  copyFileSync(args.suite, join(archiveDir, "tasks.json"));
  copyFileSync(args.models, join(archiveDir, "models.json"));
  writeJson(join(archiveDir, "pricing-snapshot.json"), models.map((m) => ({ pricingId: m.pricingId, pricing: prices.get(m.pricingId).pricing })));

  const monthlySpendAtStart = args.adapter === "openrouter" ? readMonthlySpend() : 0;
  const replayMap = args.adapter === "replay" ? loadReplay(args.replay) : null;
  const rawPath = join(archiveDir, "raw-responses.jsonl");
  const results = [];
  let runSpend = 0;
  let reserved = 0;

  // Interleave models and efforts so concurrent workers do not hammer one route.
  const jobs = [];
  for (const task of tasks) for (const model of models) for (const effort of model.efforts) jobs.push({ task, model, effort });
  let next = 0;

  async function execute({ task, model, effort }) {
    const price = prices.get(model.pricingId);
    const base = { taskId: task.id, category: task.category, difficulty: task.difficulty, pricingId: model.pricingId, openrouterId: model.openrouterId, label: model.label, effort };
    const estimated = estimateMaxCost(task, price, args.maxTokens);
    if (args.adapter === "openrouter") {
      const capHeadroom = MONTHLY_CAP_USD - monthlySpendAtStart - runSpend - reserved;
      const runHeadroom = args.budget - runSpend - reserved;
      if (estimated > capHeadroom || estimated > runHeadroom) {
        results.push({ ...base, status: "skipped_budget", correct: false, billedUsd: 0, estimatedMaxUsd: estimated });
        return;
      }
    }
    reserved += estimated;
    const started = new Date().toISOString();
    try {
      const response = args.adapter === "mock"
        ? await callMock(model, effort, task)
        : args.adapter === "replay"
        ? await callReplay(replayMap, model, effort, task)
        : await callOpenRouter(model, effort, task, env.OPENROUTER_API_KEY, args.maxTokens);
      const usage = readUsage(response.usage);
      const cost = listCost(usage, price);
      const graded = grade(response.content, task);
      const billable = args.adapter === "openrouter";
      const returned = Number(response.usage?.cost);
      const billedUsd = billable
        ? (Number.isFinite(returned) && returned >= 0 ? returned : cost.totalUsd)
        : Number(response.archivedBilledUsd || 0); // replay: report the original charge, never re-log it
      runSpend += billedUsd;
      const ledger = {
        ts: new Date().toISOString(), runId, experiment: EXPERIMENT, adapter: args.adapter, billable,
        pricingId: model.pricingId, openrouterId: model.openrouterId, effort, taskId: task.id, category: task.category,
        spendUsd: billedUsd, billedUsd, estimatedUsd: cost.totalUsd, directListPriceUsd: cost.totalUsd,
        usage: { prompt_tokens: usage.promptTokens, completion_tokens: usage.completionTokens, reasoning_tokens: usage.reasoningTokens }
      };
      if (billable) appendSpend(ledger);
      if (args.adapter !== "replay") appendFileSync(rawPath, `${JSON.stringify({ ...ledger, started, latencyMs: response.latencyMs, finishReason: response.finishReason, output: response.content, raw: response.raw })}\n`);
      results.push({
        ...base, status: "ok", correct: graded.correct, grade: graded, output: normalize(response.content),
        finishReason: response.finishReason, usage, costBreakdown: cost, listUsd: cost.totalUsd, billedUsd, latencyMs: response.latencyMs ?? null
      });
    } catch (e) {
      appendFileSync(rawPath, `${JSON.stringify({ ts: new Date().toISOString(), runId, ...base, status: "error", error: e.message, raw: e.raw || null })}\n`);
      results.push({ ...base, status: "error", error: e.message, correct: false, billedUsd: 0 });
    } finally {
      reserved -= estimated;
    }
  }
  async function worker() {
    while (next < jobs.length) await execute(jobs[next++]);
  }
  await Promise.all(Array.from({ length: Math.min(args.concurrency, jobs.length) }, worker));

  const modelIdx = new Map(models.map((m, i) => [m.pricingId, i]));
  const taskIdx = new Map(tasks.map((t, i) => [t.id, i]));
  results.sort((a, b) => modelIdx.get(a.pricingId) - modelIdx.get(b.pricingId)
    || EFFORT_ORDER.indexOf(a.effort) - EFFORT_ORDER.indexOf(b.effort)
    || taskIdx.get(a.taskId) - taskIdx.get(b.taskId));

  const cells = summarizeCells(results, models, prices);
  const modelSummaries = summarizeModels(cells, models);
  const ok = results.filter((r) => r.status === "ok");
  const result = {
    schemaVersion: 1,
    runId,
    generatedAt: new Date().toISOString(),
    experiment: EXPERIMENT,
    title: "Reasoning effort: what each effort level costs per task, and what it buys",
    adapter: args.adapter,
    methodology: {
      question: "Per-token list prices do not change with reasoning effort; the number of hidden reasoning tokens does, and they are billed as output tokens. For each model, what is the list-price cost per task at each effort level, and how does accuracy change?",
      inferenceRoute: args.adapter === "mock" ? "mock adapter" : "OpenRouter chat completions with the unified reasoning.effort parameter (reasoning text excluded from the response; still billed)",
      replayOf: args.adapter === "replay" ? args.replay : undefined,
      costBasis: "AI Pricing Guru direct-provider list prices from public/pricing.json: prompt tokens x inputPerM + (reasoning + visible output tokens) x outputPerM. No cache discounts. OpenRouter billed spend is reported separately.",
      reasoningTokenSource: "usage.completion_tokens_details.reasoning_tokens; cells where a provider does not report it are flagged reasoningReported=false and their cost still uses total completion tokens.",
      effortMapping: "OpenAI/xAI receive effort natively. For Anthropic, OpenRouter converts effort into a thinking budget proportional to max_tokens, so Anthropic results are only comparable at the same max_tokens. Gemini 3 maps effort to thinkingLevel. 'none' disables reasoning where the route allows it.",
      maxOutputTokens: args.maxTokens,
      temperature: "provider default (not sent)",
      grading: "deterministic exact/number/regex/contains/json_fields graders; no LLM judge. number/exact graders ignore markdown emphasis, code ticks, a leading $ and thousands separators; a worked explanation instead of the bare answer still fails.",
      monthlyCapUsd: MONTHLY_CAP_USD,
      runBudgetUsd: args.budget,
      concurrency: args.concurrency,
      limitations: "Single pass per task and effort; small suite (22 tasks). Reasoning-token counts vary run to run and by task type, so treat per-task costs as indicative ranges, not fixed prices. Truncated answers (finish_reason=length) count as incorrect."
    },
    suite: { id: suite.id, version: suite.version, easyTaskIds: suite.easyTaskIds || [] },
    models: models.map(({ pricingId, openrouterId, label, efforts }) => ({ pricingId, openrouterId, label, efforts })),
    tasks: tasks.map((t) => ({ id: t.id, category: t.category, difficulty: t.difficulty, prompt: t.prompt })),
    attempted: ok.length,
    correct: ok.filter((r) => r.correct).length,
    errors: results.filter((r) => r.status === "error").length,
    skippedForBudget: results.filter((r) => r.status === "skipped_budget").length,
    billedUsd: runSpend,
    listPriceCostUsd: sum(ok.map((r) => r.listUsd)),
    monthlySpendAtStartUsd: monthlySpendAtStart,
    monthlySpendAfterUsd: monthlySpendAtStart + runSpend,
    modelSummaries,
    cells,
    results,
    archiveDir,
    outputPath: args.output
  };

  writeJson(args.output, result);
  writeJson(join(archiveDir, "results.json"), result);
  if (args.publicOutput) writeJson(args.publicOutput, result);
  if (args.adapter === "openrouter") {
    writeJson(LAST_RUN, { ts: result.generatedAt, experiment: EXPERIMENT, spend: runSpend, outcome: `${result.correct}/${result.attempted} correct across ${models.length} models, ${cells.length} model-effort cells, ${tasks.length} tasks`, runId, output: args.output, archiveDir });
    if (monthlySpendAtStart < WARN_AT_USD && monthlySpendAtStart + runSpend >= WARN_AT_USD) {
      if (!args.noTelegram) await notifyGate({ level: "hot", source: "labs", text: `AI Pricing Guru Labs spend warning: monthly spend crossed ${fmt(WARN_AT_USD, 2)}. Current: ${fmt(monthlySpendAtStart + runSpend, 2)} / ${fmt(MONTHLY_CAP_USD, 2)}.` });
    }
    if (!args.noTelegram) await notifyGate({ level: "weekly", source: "labs", text: telegramText(result) });
  }

  console.log(JSON.stringify({
    runId, adapter: args.adapter, output: args.output, archiveDir,
    billedUsd: Number(runSpend.toFixed(4)), monthlySpendAfterUsd: Number((monthlySpendAtStart + runSpend).toFixed(2)),
    attempted: result.attempted, correct: result.correct, errors: result.errors, skippedForBudget: result.skippedForBudget,
    models: modelSummaries.map((m) => ({
      label: m.label,
      efforts: m.efforts.map((e) => `${e.effort}: ${fmt(e.avgListUsdPerTask, 5)}/task, x${e.costMultiplierVsBaseline?.toFixed(2) ?? "?"}, acc ${e.accuracy == null ? "n/a" : (e.accuracy * 100).toFixed(0) + "%"}, hard ${e.hardAccuracy == null ? "n/a" : (e.hardAccuracy * 100).toFixed(0) + "%"}, reasoning ${e.avgReasoningTokens?.toFixed(0) ?? "?"} tok${e.truncatedCount ? `, ${e.truncatedCount} truncated` : ""}${e.errorCount ? `, ${e.errorCount} errors` : ""}`),
      unavailable: m.unavailableEfforts
    }))
  }, null, 2));
}

main().catch((e) => {
  console.error(e.stack || e.message);
  process.exit(1);
});