{ "runId": "2026-09-07T06-50-03-632Z-cost-per-task-openrouter", "generatedAt": "2026-09-07T06:58:43.406Z", "experiment": "cost-per-task", "adapter": "openrouter", "methodology": { "inferenceRoute": "OpenRouter chat completions", "costBasis": "AI Pricing Guru direct-provider list prices from /opt/aipricing/repo/public/pricing.json", "grading": "deterministic exact/number/regex/contains/json_fields graders; no LLM judge", "monthlyCapUsd": 100, "runBudgetUsd": 5, "maxOutputTokens": 1024, "concurrency": 4 }, "suite": { "id": "cost-per-task-v2", "version": 2 }, "models": [ { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna" }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra" }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol" }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5" }, { "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3" }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8" }, { "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5" }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5" }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5" }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash" }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite" }, { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro" }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash" }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2" }, { "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6" }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5" } ], "tasks": [ { "id": "math-001", "category": "reasoning", "prompt": "Return only the integer answer. A batch job processes 18 files per minute for 7 minutes, then 12 files per minute for 5 minutes. How many files total?" }, { "id": "math-002", "category": "reasoning", "prompt": "Return only the integer answer. A model costs $0.30 per 1M input tokens and $2.50 per 1M output tokens. What is the cost in cents for 100,000 input tokens and 20,000 output tokens?" }, { "id": "math-003", "category": "reasoning", "prompt": "Return only the integer answer. If a 64,000-token context window is filled to 75%, how many tokens are used?" }, { "id": "math-004", "category": "reasoning", "prompt": "Return only the integer answer. A provider raises output price from $4/M to $6/M. What is the percent increase?" }, { "id": "math-005", "category": "reasoning", "prompt": "Return only the integer answer. A cache hit costs 10% of normal input price. Normal input is $2/M. What is the cache-hit price in cents per 1M tokens?" }, { "id": "math-006", "category": "reasoning", "prompt": "Return only the integer answer. A crawler finds 9 new models Monday, 14 Tuesday, and removes 5 duplicates. How many unique new models remain?" }, { "id": "math-007", "category": "reasoning", "prompt": "Return only the integer answer. A benchmark has 50 tasks. A model gets 37 correct. What is the accuracy percentage rounded to the nearest whole number?" }, { "id": "math-008", "category": "reasoning", "prompt": "Return only the decimal answer with no currency symbol. If a run spends $0.42 and gets 14 correct answers, what is dollars per correct answer?" }, { "id": "extract-001", "category": "extraction", "prompt": "Return only JSON with numeric values for prices and context, without currency symbols or units. Text: 'Gemini Flash: input $0.30/M, output $2.50/M, context 1,048,576 tokens.' Extract fields model, inputPerM, outputPerM, context." }, { "id": "extract-002", "category": "extraction", "prompt": "Return only JSON. Text: 'Z.ai GLM-5.2 charges $1.40 per million input tokens and $4.40 per million output tokens.' Extract provider, model, inputPerM, outputPerM." }, { "id": "extract-003", "category": "extraction", "prompt": "Return only the single word immediately before 'plan'. Sentence: 'For heavy coding, the Pro plan is cheaper than per-token API use after roughly 35 million output tokens.'" }, { "id": "extract-004", "category": "extraction", "prompt": "Return only JSON. Log line: '2026-07-20 model=deepseek-v4-flash prompt=120 completion=38 latency_ms=910 ok=true'. Extract model, prompt, completion, latency_ms, ok." }, { "id": "extract-005", "category": "extraction", "prompt": "Return only a comma-separated list of model IDs in their original order. Text: 'Tested: gpt-5.6-luna; claude-sonnet-5; gemini-2.5-flash.'" }, { "id": "extract-006", "category": "extraction", "prompt": "Return only JSON. Text: 'Anthropic Claude Haiku 4.5: $1 input, $5 output, active.' Extract provider, model, status." }, { "id": "classify-001", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'OpenAI cuts GPT-5.6 Luna output token price by 20%'." }, { "id": "classify-002", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'DeepSeek introduces V4 Flash for low-latency chat apps'." }, { "id": "classify-003", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Claude API errors spike across us-east for 42 minutes'." }, { "id": "classify-004", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Step-by-step guide to add streaming responses to a chatbot'." }, { "id": "classify-005", "category": "classification", "prompt": "Return only yes or no. Query: 'cheapest ai api per token'. Is this query primarily cost/comparison intent?" }, { "id": "classify-006", "category": "classification", "prompt": "Return only yes or no. Query: 'what is a transformer neural network'. Is this query primarily pricing intent?" }, { "id": "summ-001", "category": "summarization", "prompt": "Summarize in exactly 8 words and include these exact words: cheaper, extraction, failed, arithmetic. Text: 'The cheaper model answered all extraction tasks correctly but failed most multi-step arithmetic, making it a good fit for structured parsing rather than reasoning.'" }, { "id": "summ-002", "category": "summarization", "prompt": "Return these three search phrases exactly as written, lowercase and separated by commas with no extra text: ai api pricing, cheapest ai api, token calculator. Text: 'Rising searches include ai api pricing, cheapest ai api, and token calculator, all pointing toward cost-conscious developer intent.'" }, { "id": "summ-003", "category": "summarization", "prompt": "Return one sentence under 12 words and include the exact words verbosity and cost. Text: 'A model with high accuracy but very high output verbosity can lose on dollars per correct answer because every extra token compounds cost.'" }, { "id": "summ-004", "category": "summarization", "prompt": "Return only a title under 7 words that includes the exact words cost and correct. Text: 'We compare leading AI models by how much each correct benchmark answer costs, not just accuracy.'" }, { "id": "summ-005", "category": "summarization", "prompt": "Return exactly one sentence that includes OpenRouter and the exact phrase list prices. Text: 'OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.'" }, { "id": "code-001", "category": "coding", "prompt": "Return only the output. JavaScript: const xs=[3,1,4,1,5]; console.log(xs.filter(x=>x>2).reduce((a,b)=>a+b,0));" }, { "id": "code-002", "category": "coding", "prompt": "Return only the output. Python: prices={'in':0.3,'out':2.5}; print(round(prices['in']*2 + prices['out']*0.1, 2))" }, { "id": "code-003", "category": "coding", "prompt": "Return only the missing JavaScript expression. Complete: const cost = (inputTokens * ___ + outputTokens * outputPerM) / 1_000_000;" }, { "id": "code-004", "category": "coding", "prompt": "Return only JSON. For rows [{ok:true,cost:0.02},{ok:false,cost:0.03},{ok:true,cost:0.01}], compute correct, totalCost, costPerCorrect." }, { "id": "code-005", "category": "coding", "prompt": "Return only the output. JavaScript: console.log(['gpt','claude','glm'].map(s=>s.length).join('-'));" }, { "id": "code-006", "category": "coding", "prompt": "Return only the called function name immediately after await. Snippet: async function call(){ const r = await fetch(url, opts); return r.json(); }" }, { "id": "logic-001", "category": "instruction", "prompt": "Return only the third item alphabetically: Luna, Flash, Sonnet, Grok." }, { "id": "logic-002", "category": "instruction", "prompt": "Return only the word that appears twice: token, price, model, token, output." }, { "id": "logic-003", "category": "instruction", "prompt": "Return only the reversed string: glm-5.2" }, { "id": "logic-004", "category": "instruction", "prompt": "Return only the uppercase acronym from: cost per correct answer." }, { "id": "logic-005", "category": "instruction", "prompt": "Return only a comma-separated list of the two providers with names starting with G: OpenAI, Google, Anthropic, Groq, Mistral." }, { "id": "logic-006", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {\"model\":\"gpt\",\"cost\":0.01}" }, { "id": "logic-007", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {model:\"gpt\",cost:0.01}" }, { "id": "logic-008", "category": "instruction", "prompt": "Return only the smallest price: $1.25/M, $0.30/M, $2.00/M, $0.14/M." }, { "id": "logic-009", "category": "instruction", "prompt": "Return only the final model ID. Chain: start=gpt-5; replace gpt with glm; append .2; prefix z-ai-." }, { "id": "logic-010", "category": "instruction", "prompt": "Return only the count of unique providers: OpenAI, Anthropic, OpenAI, Google, Z.ai, Google." }, { "id": "pricing-001", "category": "pricing", "prompt": "Return only cheaper or expensive. Model A costs $0.10/M input and $0.30/M output. Model B costs $0.30/M input and $2.50/M output. For any positive input/output workload, Model A is what relative to Model B?" }, { "id": "pricing-002", "category": "pricing", "prompt": "Return only the model name. A got 10 correct for $0.20. B got 8 correct for $0.08. Which has lower dollars per correct answer?" }, { "id": "pricing-003", "category": "pricing", "prompt": "Return only the integer answer. If output tokens are 4x more expensive than input tokens, how many input-token equivalents is 250 output tokens?" }, { "id": "pricing-004", "category": "pricing", "prompt": "Return only JSON. A run used 1000 prompt tokens and 500 completion tokens. Prices are $1/M input and $6/M output. Compute inputCost, outputCost, totalCost in dollars." }, { "id": "pricing-005", "category": "pricing", "prompt": "Return only yes or no. If a model is free for input but charges for output, can a verbose wrong answer still cost money?" }, { "id": "pricing-006", "category": "pricing", "prompt": "Return only the ratio as N:1. Output costs $15/M and input costs $3/M." }, { "id": "pricing-007", "category": "pricing", "prompt": "Return only the integer answer. A $5 pilot budget has already spent $1.75. How many cents remain?" }, { "id": "pricing-008", "category": "pricing", "prompt": "Return only the best metric name: cost per prompt, cost per correct answer, or total tokens. We need to compare accuracy and spend together." } ], "spendUsd": 0.6732161827340007, "billedUsd": 0.6732161827340007, "listPriceCostUsd": 0.71426765, "monthlySpendAtStartUsd": 0.6989654182599999, "monthlySpendAfterUsd": 1.3721816009940007, "attempted": 784, "correct": 771, "skippedForBudget": 0, "modelSummaries": [ { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.0024492, "billedUsd": 0.0024492, "costPerCorrectUsd": 0.00004998367346938775, "promptTokens": 3426, "completionTokens": 1470, "avgCompletionTokens": 30 }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.019956, "billedUsd": 0.019956, "costPerCorrectUsd": 0.00040726530612244903, "promptTokens": 3426, "completionTokens": 1092, "avgCompletionTokens": 22.285714285714285 }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.020853999999999998, "billedUsd": 0.020853999999999998, "costPerCorrectUsd": 0.0004255918367346938, "promptTokens": 5232, "completionTokens": 1039, "avgCompletionTokens": 21.20408163265306 }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.029803999999999994, "billedUsd": 0.014901999999999997, "costPerCorrectUsd": 0.0006082448979591835, "promptTokens": 3426, "completionTokens": 805, "avgCompletionTokens": 16.428571428571427 }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.0326944, "billedUsd": 0.022646422, "costPerCorrectUsd": 0.0006672326530612245, "promptTokens": 3572, "completionTokens": 6294, "avgCompletionTokens": 128.44897959183675 }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.051867750000000004, "billedUsd": 0.05186775, "costPerCorrectUsd": 0.0010585255102040817, "promptTokens": 3042, "completionTokens": 13223, "avgCompletionTokens": 269.85714285714283 }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.06707600000000001, "billedUsd": 0.05946000000000001, "costPerCorrectUsd": 0.0013688979591836737, "promptTokens": 13216, "completionTokens": 6774, "avgCompletionTokens": 138.24489795918367 }, { "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.08923500000000001, "billedUsd": 0.08923500000000001, "costPerCorrectUsd": 0.001821122448979592, "promptTokens": 5232, "completionTokens": 2523, "avgCompletionTokens": 51.48979591836735 }, { "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.110798, "billedUsd": 0.10311799999999997, "costPerCorrectUsd": 0.002261183673469388, "promptTokens": 13216, "completionTokens": 14061, "avgCompletionTokens": 286.9591836734694 }, { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.0092433, "billedUsd": 0.014308838379999998, "costPerCorrectUsd": 0.00019256875, "promptTokens": 3547, "completionTokens": 3486, "avgCompletionTokens": 71.14285714285714 }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.04138499999999999, "billedUsd": 0.04138499999999999, "costPerCorrectUsd": 0.0008621874999999998, "promptTokens": 5232, "completionTokens": 609, "avgCompletionTokens": 12.428571428571429 }, { "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.11093400000000003, "billedUsd": 0.10667030999999999, "costPerCorrectUsd": 0.0023111250000000007, "promptTokens": 8683, "completionTokens": 5659, "avgCompletionTokens": 115.48979591836735 }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.11572999999999999, "billedUsd": 0.11572999999999999, "costPerCorrectUsd": 0.0024110416666666664, "promptTokens": 5133, "completionTokens": 1288, "avgCompletionTokens": 26.285714285714285 }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0025776, "billedUsd": 0.0025776, "costPerCorrectUsd": 0.000056034782608695655, "promptTokens": 3042, "completionTokens": 666, "avgCompletionTokens": 13.591836734693878 }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0030074000000000012, "billedUsd": 0.001400062354, "costPerCorrectUsd": 0.00006537826086956524, "promptTokens": 3623, "completionTokens": 3349, "avgCompletionTokens": 68.34693877551021 }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0066560000000000005, "billedUsd": 0.0066560000000000005, "costPerCorrectUsd": 0.00014469565217391304, "promptTokens": 3486, "completionTokens": 634, "avgCompletionTokens": 12.938775510204081 } ], "results": [ { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00005160000000000001, "billedUsd": 0.0000516, "estimatedUsd": 0.00005160000000000001, "usage": { "prompt_tokens": 72, "completion_tokens": 31, "total_tokens": 103 }, "latencyMs": 2998 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0000824, "billedUsd": 0.0000824, "estimatedUsd": 0.0000824, "usage": { "prompt_tokens": 88, "completion_tokens": 54, "total_tokens": 142 }, "latencyMs": 2611 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000044999999999999996, "billedUsd": 0.000045, "estimatedUsd": 0.000044999999999999996, "usage": { "prompt_tokens": 63, "completion_tokens": 27, "total_tokens": 90 }, "latencyMs": 3058 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0000424, "billedUsd": 0.0000424, "estimatedUsd": 0.0000424, "usage": { "prompt_tokens": 62, "completion_tokens": 25, "total_tokens": 87 }, "latencyMs": 1979 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000051, "billedUsd": 0.000051, "estimatedUsd": 0.000051, "usage": { "prompt_tokens": 75, "completion_tokens": 30, "total_tokens": 105 }, "latencyMs": 1817 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1013 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1259 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000022, "billedUsd": 0.000022, "estimatedUsd": 0.000022, "usage": { "prompt_tokens": 68, "completion_tokens": 7, "total_tokens": 75 }, "latencyMs": 949 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.000059, "billedUsd": 0.000059, "estimatedUsd": 0.000059, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1046 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.00011499999999999999, "billedUsd": 0.000115, "estimatedUsd": 0.00011499999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 82, "total_tokens": 165 }, "latencyMs": 2201 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000020200000000000003, "billedUsd": 0.0000202, "estimatedUsd": 0.000020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 881 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00008379999999999999, "billedUsd": 0.0000838, "estimatedUsd": 0.00008379999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 56, "total_tokens": 139 }, "latencyMs": 1918 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.0000498, "billedUsd": 0.0000498, "estimatedUsd": 0.0000498, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 935 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.0000764, "billedUsd": 0.0000764, "estimatedUsd": 0.0000764, "usage": { "prompt_tokens": 70, "completion_tokens": 52, "total_tokens": 122 }, "latencyMs": 1199 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000020200000000000003, "billedUsd": 0.0000202, "estimatedUsd": 0.000020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 899 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 960 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000204, "billedUsd": 0.0000204, "estimatedUsd": 0.0000204, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 965 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000019200000000000003, "billedUsd": 0.0000192, "estimatedUsd": 0.000019200000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 1006 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000018200000000000002, "billedUsd": 0.0000182, "estimatedUsd": 0.000018200000000000002, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 842 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0000178, "billedUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 984 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded; failed arithmetic; best for parsing.", "spendUsd": 0.00019759999999999998, "billedUsd": 0.0001976, "estimatedUsd": 0.00019759999999999998, "usage": { "prompt_tokens": 88, "completion_tokens": 150, "total_tokens": 238 }, "latencyMs": 2274 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000354, "billedUsd": 0.0000354, "estimatedUsd": 0.0000354, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 1022 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "High verbosity increases cost despite accuracy.", "spendUsd": 0.00009620000000000001, "billedUsd": 0.0000962, "estimatedUsd": 0.00009620000000000001, "usage": { "prompt_tokens": 79, "completion_tokens": 67, "total_tokens": 146 }, "latencyMs": 2214 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The cost of a correct answer", "spendUsd": 0.0001192, "billedUsd": 0.0001192, "estimatedUsd": 0.0001192, "usage": { "prompt_tokens": 74, "completion_tokens": 87, "total_tokens": 161 }, "latencyMs": 2214 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0001026, "billedUsd": 0.0001026, "estimatedUsd": 0.0001026, "usage": { "prompt_tokens": 87, "completion_tokens": 71, "total_tokens": 158 }, "latencyMs": 1348 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000021400000000000002, "billedUsd": 0.0000214, "estimatedUsd": 0.000021400000000000002, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 1228 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0000626, "billedUsd": 0.0000626, "estimatedUsd": 0.0000626, "usage": { "prompt_tokens": 79, "completion_tokens": 39, "total_tokens": 118 }, "latencyMs": 1874 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0000224, "billedUsd": 0.0000224, "estimatedUsd": 0.0000224, "usage": { "prompt_tokens": 70, "completion_tokens": 7, "total_tokens": 77 }, "latencyMs": 1242 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0000856, "billedUsd": 0.0000856, "estimatedUsd": 0.0000856, "usage": { "prompt_tokens": 80, "completion_tokens": 58, "total_tokens": 138 }, "latencyMs": 2007 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000023599999999999998, "billedUsd": 0.0000236, "estimatedUsd": 0.000023599999999999998, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 942 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000194, "billedUsd": 0.0000194, "estimatedUsd": 0.0000194, "usage": { "prompt_tokens": 67, "completion_tokens": 5, "total_tokens": 72 }, "latencyMs": 955 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000048, "billedUsd": 0.000048, "estimatedUsd": 0.000048, "usage": { "prompt_tokens": 54, "completion_tokens": 31, "total_tokens": 85 }, "latencyMs": 1932 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000016800000000000002, "billedUsd": 0.0000168, "estimatedUsd": 0.000016800000000000002, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1033 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0000214, "billedUsd": 0.0000214, "estimatedUsd": 0.0000214, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 929 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000016800000000000002, "billedUsd": 0.0000168, "estimatedUsd": 0.000016800000000000002, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 1174 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000045799999999999995, "billedUsd": 0.0000458, "estimatedUsd": 0.000045799999999999995, "usage": { "prompt_tokens": 67, "completion_tokens": 27, "total_tokens": 94 }, "latencyMs": 1651 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000063, "billedUsd": 0.000063, "estimatedUsd": 0.000063, "usage": { "prompt_tokens": 57, "completion_tokens": 43, "total_tokens": 100 }, "latencyMs": 1864 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0000462, "billedUsd": 0.0000462, "estimatedUsd": 0.0000462, "usage": { "prompt_tokens": 57, "completion_tokens": 29, "total_tokens": 86 }, "latencyMs": 2420 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000024, "billedUsd": 0.000024, "estimatedUsd": 0.000024, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 913 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00008740000000000001, "billedUsd": 0.0000874, "estimatedUsd": 0.00008740000000000001, "usage": { "prompt_tokens": 65, "completion_tokens": 62, "total_tokens": 127 }, "latencyMs": 1267 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000018, "billedUsd": 0.000018, "estimatedUsd": 0.000018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1136 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000256, "billedUsd": 0.0000256, "estimatedUsd": 0.0000256, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 1116 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0000612, "billedUsd": 0.0000612, "estimatedUsd": 0.0000612, "usage": { "prompt_tokens": 72, "completion_tokens": 39, "total_tokens": 111 }, "latencyMs": 2028 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000444, "billedUsd": 0.0000444, "estimatedUsd": 0.0000444, "usage": { "prompt_tokens": 66, "completion_tokens": 26, "total_tokens": 92 }, "latencyMs": 1826 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0000996, "billedUsd": 0.0000996, "estimatedUsd": 0.0000996, "usage": { "prompt_tokens": 78, "completion_tokens": 70, "total_tokens": 148 }, "latencyMs": 1905 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000186, "billedUsd": 0.0000186, "estimatedUsd": 0.0000186, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 1081 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000714, "billedUsd": 0.0000714, "estimatedUsd": 0.0000714, "usage": { "prompt_tokens": 57, "completion_tokens": 50, "total_tokens": 107 }, "latencyMs": 1896 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000048, "billedUsd": 0.000048, "estimatedUsd": 0.000048, "usage": { "prompt_tokens": 60, "completion_tokens": 30, "total_tokens": 90 }, "latencyMs": 2110 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00005379999999999999, "billedUsd": 0.0000538, "estimatedUsd": 0.00005379999999999999, "usage": { "prompt_tokens": 65, "completion_tokens": 34, "total_tokens": 99 }, "latencyMs": 1762 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 2098 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0007160000000000001, "billedUsd": 0.000716, "estimatedUsd": 0.0007160000000000001, "usage": { "prompt_tokens": 88, "completion_tokens": 45, "total_tokens": 133 }, "latencyMs": 2223 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00035400000000000004, "billedUsd": 0.000354, "estimatedUsd": 0.00035400000000000004, "usage": { "prompt_tokens": 63, "completion_tokens": 19, "total_tokens": 82 }, "latencyMs": 1535 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000328, "billedUsd": 0.000328, "estimatedUsd": 0.000328, "usage": { "prompt_tokens": 62, "completion_tokens": 17, "total_tokens": 79 }, "latencyMs": 1337 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0004739999999999999, "billedUsd": 0.000474, "estimatedUsd": 0.0004739999999999999, "usage": { "prompt_tokens": 75, "completion_tokens": 27, "total_tokens": 102 }, "latencyMs": 2175 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1158 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1335 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0005200000000000001, "billedUsd": 0.00052, "estimatedUsd": 0.0005200000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 32, "total_tokens": 100 }, "latencyMs": 1722 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.00059, "billedUsd": 0.00059, "estimatedUsd": 0.00059, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1183 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.0005859999999999999, "billedUsd": 0.000586, "estimatedUsd": 0.0005859999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1134 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000406, "billedUsd": 0.000406, "estimatedUsd": 0.000406, "usage": { "prompt_tokens": 71, "completion_tokens": 22, "total_tokens": 93 }, "latencyMs": 1456 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1036 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000498, "billedUsd": 0.000498, "estimatedUsd": 0.000498, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1242 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.000776, "billedUsd": 0.000776, "estimatedUsd": 0.000776, "usage": { "prompt_tokens": 70, "completion_tokens": 53, "total_tokens": 123 }, "latencyMs": 1619 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00020200000000000003, "billedUsd": 0.000202, "estimatedUsd": 0.00020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 969 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 874 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 901 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 941 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 979 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000178, "billedUsd": 0.000178, "estimatedUsd": 0.000178, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1514 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper model excelled extraction, failed arithmetic reasoning tasks.", "spendUsd": 0.0009559999999999998, "billedUsd": 0.000956, "estimatedUsd": 0.0009559999999999998, "usage": { "prompt_tokens": 88, "completion_tokens": 65, "total_tokens": 153 }, "latencyMs": 2351 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000354, "billedUsd": 0.000354, "estimatedUsd": 0.000354, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 944 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity can raise cost despite accuracy.", "spendUsd": 0.00030199999999999997, "billedUsd": 0.000302, "estimatedUsd": 0.00030199999999999997, "usage": { "prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91 }, "latencyMs": 1058 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost per Correct AI Answer", "spendUsd": 0.000256, "billedUsd": 0.000256, "estimatedUsd": 0.000256, "usage": { "prompt_tokens": 74, "completion_tokens": 9, "total_tokens": 83 }, "latencyMs": 1048 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000618, "billedUsd": 0.000618, "estimatedUsd": 0.000618, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 1079 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00035800000000000003, "billedUsd": 0.000358, "estimatedUsd": 0.00035800000000000003, "usage": { "prompt_tokens": 77, "completion_tokens": 17, "total_tokens": 94 }, "latencyMs": 1252 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0006739999999999999, "billedUsd": 0.000674, "estimatedUsd": 0.0006739999999999999, "usage": { "prompt_tokens": 79, "completion_tokens": 43, "total_tokens": 122 }, "latencyMs": 2299 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0005, "billedUsd": 0.0005, "estimatedUsd": 0.0005, "usage": { "prompt_tokens": 70, "completion_tokens": 30, "total_tokens": 100 }, "latencyMs": 1923 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.000784, "billedUsd": 0.000784, "estimatedUsd": 0.000784, "usage": { "prompt_tokens": 80, "completion_tokens": 52, "total_tokens": 132 }, "latencyMs": 2331 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000428, "billedUsd": 0.000428, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 64, "completion_tokens": 25, "total_tokens": 89 }, "latencyMs": 1323 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00041, "billedUsd": 0.00041, "estimatedUsd": 0.00041, "usage": { "prompt_tokens": 67, "completion_tokens": 23, "total_tokens": 90 }, "latencyMs": 1535 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000396, "billedUsd": 0.000396, "estimatedUsd": 0.000396, "usage": { "prompt_tokens": 54, "completion_tokens": 24, "total_tokens": 78 }, "latencyMs": 1360 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00016800000000000002, "billedUsd": 0.000168, "estimatedUsd": 0.00016800000000000002, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1028 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000778, "billedUsd": 0.000778, "estimatedUsd": 0.000778, "usage": { "prompt_tokens": 47, "completion_tokens": 57, "total_tokens": 104 }, "latencyMs": 2451 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00016800000000000002, "billedUsd": 0.000168, "estimatedUsd": 0.00016800000000000002, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 1058 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1006 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00035400000000000004, "billedUsd": 0.000354, "estimatedUsd": 0.00035400000000000004, "usage": { "prompt_tokens": 57, "completion_tokens": 20, "total_tokens": 77 }, "latencyMs": 2126 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00039, "billedUsd": 0.00039, "estimatedUsd": 0.00039, "usage": { "prompt_tokens": 57, "completion_tokens": 23, "total_tokens": 80 }, "latencyMs": 1814 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00024, "billedUsd": 0.00024, "estimatedUsd": 0.00024, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1090 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000814, "billedUsd": 0.000814, "estimatedUsd": 0.000814, "usage": { "prompt_tokens": 65, "completion_tokens": 57, "total_tokens": 122 }, "latencyMs": 1904 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00018, "billedUsd": 0.00018, "estimatedUsd": 0.00018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1646 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000256, "billedUsd": 0.000256, "estimatedUsd": 0.000256, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 1315 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00037200000000000004, "billedUsd": 0.000372, "estimatedUsd": 0.00037200000000000004, "usage": { "prompt_tokens": 72, "completion_tokens": 19, "total_tokens": 91 }, "latencyMs": 1154 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1053 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0009480000000000001, "billedUsd": 0.000948, "estimatedUsd": 0.0009480000000000001, "usage": { "prompt_tokens": 78, "completion_tokens": 66, "total_tokens": 144 }, "latencyMs": 3569 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00018600000000000002, "billedUsd": 0.000186, "estimatedUsd": 0.00018600000000000002, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 1041 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00043799999999999997, "billedUsd": 0.000438, "estimatedUsd": 0.00043799999999999997, "usage": { "prompt_tokens": 57, "completion_tokens": 27, "total_tokens": 84 }, "latencyMs": 1994 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00018, "billedUsd": 0.00018, "estimatedUsd": 0.00018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1827 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00046599999999999994, "billedUsd": 0.000466, "estimatedUsd": 0.00046599999999999994, "usage": { "prompt_tokens": 65, "completion_tokens": 28, "total_tokens": 93 }, "latencyMs": 3065 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000388, "billedUsd": 0.000194, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 3068 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001292, "billedUsd": 0.000646, "estimatedUsd": 0.001292, "usage": { "prompt_tokens": 88, "completion_tokens": 47, "total_tokens": 135 }, "latencyMs": 2882 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000372, "billedUsd": 0.000186, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 63, "completion_tokens": 6, "total_tokens": 69 }, "latencyMs": 1535 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000348, "billedUsd": 0.000174, "estimatedUsd": 0.000348, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 963 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00039999999999999996, "billedUsd": 0.0002, "estimatedUsd": 0.00039999999999999996, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 1222 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000372, "billedUsd": 0.000186, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1007 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000372, "billedUsd": 0.000186, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 861 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000412, "billedUsd": 0.000206, "estimatedUsd": 0.000412, "usage": { "prompt_tokens": 68, "completion_tokens": 7, "total_tokens": 75 }, "latencyMs": 1342 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0010479999999999999, "billedUsd": 0.000524, "estimatedUsd": 0.0010479999999999999, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1367 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.001032, "billedUsd": 0.000516, "estimatedUsd": 0.001032, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 3215 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000384, "billedUsd": 0.000192, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1577 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.000972, "billedUsd": 0.000486, "estimatedUsd": 0.000972, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1357 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000884, "billedUsd": 0.000442, "estimatedUsd": 0.000884, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1262 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00076, "billedUsd": 0.00038, "estimatedUsd": 0.00076, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1982 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000384, "billedUsd": 0.000192, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1304 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000372, "billedUsd": 0.000186, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1125 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000384, "billedUsd": 0.000192, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1080 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000364, "billedUsd": 0.000182, "estimatedUsd": 0.000364, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 1023 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000344, "billedUsd": 0.000172, "estimatedUsd": 0.000344, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 933 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000336, "billedUsd": 0.000168, "estimatedUsd": 0.000336, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 989 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model mastered extraction but failed arithmetic.", "spendUsd": 0.002752, "billedUsd": 0.001376, "estimatedUsd": 0.002752, "usage": { "prompt_tokens": 88, "completion_tokens": 120, "total_tokens": 208 }, "latencyMs": 2615 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000652, "billedUsd": 0.000326, "estimatedUsd": 0.000652, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 1055 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity increases cost, undermining accuracy gains per correct answer.", "spendUsd": 0.001856, "billedUsd": 0.000928, "estimatedUsd": 0.001856, "usage": { "prompt_tokens": 79, "completion_tokens": 77, "total_tokens": 156 }, "latencyMs": 2621 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "cost per correct AI answer", "spendUsd": 0.001716, "billedUsd": 0.000858, "estimatedUsd": 0.001716, "usage": { "prompt_tokens": 74, "completion_tokens": 71, "total_tokens": 145 }, "latencyMs": 3322 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.001088, "billedUsd": 0.000544, "estimatedUsd": 0.001088, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 2323 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000408, "billedUsd": 0.000204, "estimatedUsd": 0.000408, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 1035 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00045599999999999997, "billedUsd": 0.000228, "estimatedUsd": 0.00045599999999999997, "usage": { "prompt_tokens": 79, "completion_tokens": 7, "total_tokens": 86 }, "latencyMs": 1134 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00076, "billedUsd": 0.00038, "estimatedUsd": 0.00076, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1281 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0008, "billedUsd": 0.0004, "estimatedUsd": 0.0008, "usage": { "prompt_tokens": 80, "completion_tokens": 24, "total_tokens": 104 }, "latencyMs": 1223 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00043599999999999997, "billedUsd": 0.000218, "estimatedUsd": 0.00043599999999999997, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1405 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000368, "billedUsd": 0.000184, "estimatedUsd": 0.000368, "usage": { "prompt_tokens": 67, "completion_tokens": 5, "total_tokens": 72 }, "latencyMs": 1369 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000336, "billedUsd": 0.000168, "estimatedUsd": 0.000336, "usage": { "prompt_tokens": 54, "completion_tokens": 6, "total_tokens": 60 }, "latencyMs": 941 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000316, "billedUsd": 0.000158, "estimatedUsd": 0.000316, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1244 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000388, "billedUsd": 0.000194, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 1015 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000312, "billedUsd": 0.000156, "estimatedUsd": 0.000312, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 1057 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000428, "billedUsd": 0.000214, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1667 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000328, "billedUsd": 0.000164, "estimatedUsd": 0.000328, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 1967 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000328, "billedUsd": 0.000164, "estimatedUsd": 0.000328, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 1469 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00044400000000000006, "billedUsd": 0.000222, "estimatedUsd": 0.00044400000000000006, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 2219 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0005, "billedUsd": 0.00025, "estimatedUsd": 0.0005, "usage": { "prompt_tokens": 65, "completion_tokens": 12, "total_tokens": 77 }, "latencyMs": 2736 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00034, "billedUsd": 0.00017, "estimatedUsd": 0.00034, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1256 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000488, "billedUsd": 0.000244, "estimatedUsd": 0.000488, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 1275 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000388, "billedUsd": 0.000194, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 1485 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000384, "billedUsd": 0.000192, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 946 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.000832, "billedUsd": 0.000416, "estimatedUsd": 0.000832, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 2011 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000352, "billedUsd": 0.000176, "estimatedUsd": 0.000352, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 971 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000368, "billedUsd": 0.000184, "estimatedUsd": 0.000368, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 1694 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00034, "billedUsd": 0.00017, "estimatedUsd": 0.00034, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1180 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00041999999999999996, "billedUsd": 0.00021, "estimatedUsd": 0.00041999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 8, "total_tokens": 73 }, "latencyMs": 1315 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00216, "billedUsd": 0.00216, "estimatedUsd": 0.00216, "usage": { "prompt_tokens": 106, "completion_tokens": 22, "total_tokens": 128 }, "latencyMs": 7056 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0042, "billedUsd": 0.0042, "estimatedUsd": 0.0042, "usage": { "prompt_tokens": 125, "completion_tokens": 59, "total_tokens": 184 }, "latencyMs": 3381 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0011, "billedUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 2412 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0010500000000000002, "billedUsd": 0.00105, "estimatedUsd": 0.0010500000000000002, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 3116 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0024000000000000002, "billedUsd": 0.0024, "estimatedUsd": 0.0024000000000000002, "usage": { "prompt_tokens": 110, "completion_tokens": 26, "total_tokens": 136 }, "latencyMs": 4127 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00123, "billedUsd": 0.00123, "estimatedUsd": 0.00123, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 3204 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00124, "billedUsd": 0.00124, "estimatedUsd": 0.00124, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 3299 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00129, "billedUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 3375 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.00379, "billedUsd": 0.00379, "estimatedUsd": 0.00379, "usage": { "prompt_tokens": 144, "completion_tokens": 47, "total_tokens": 191 }, "latencyMs": 3340 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00377, "billedUsd": 0.00377, "estimatedUsd": 0.00377, "usage": { "prompt_tokens": 127, "completion_tokens": 50, "total_tokens": 177 }, "latencyMs": 3230 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00141, "billedUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 3070 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.00422, "billedUsd": 0.00422, "estimatedUsd": 0.00422, "usage": { "prompt_tokens": 132, "completion_tokens": 58, "total_tokens": 190 }, "latencyMs": 3885 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0028200000000000005, "billedUsd": 0.00282, "estimatedUsd": 0.0028200000000000005, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 3120 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.005019999999999999, "billedUsd": 0.00502, "estimatedUsd": 0.005019999999999999, "usage": { "prompt_tokens": 107, "completion_tokens": 79, "total_tokens": 186 }, "latencyMs": 3417 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 3039 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00129, "billedUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 3157 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0012399999999999998, "billedUsd": 0.00124, "estimatedUsd": 0.0012399999999999998, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 3137 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 2931 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00115, "billedUsd": 0.00115, "estimatedUsd": 0.00115, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 1903 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00107, "billedUsd": 0.00107, "estimatedUsd": 0.00107, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 3244 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded at extraction but failed arithmetic.", "spendUsd": 0.00835, "billedUsd": 0.00835, "estimatedUsd": 0.00835, "usage": { "prompt_tokens": 150, "completion_tokens": 137, "total_tokens": 287 }, "latencyMs": 4409 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0026, "billedUsd": 0.0026, "estimatedUsd": 0.0026, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 3551 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0021, "billedUsd": 0.0021, "estimatedUsd": 0.0021, "usage": { "prompt_tokens": 125, "completion_tokens": 17, "total_tokens": 142 }, "latencyMs": 3185 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost per Correct Answer in AI", "spendUsd": 0.00189, "billedUsd": 0.00189, "estimatedUsd": 0.00189, "usage": { "prompt_tokens": 114, "completion_tokens": 15, "total_tokens": 129 }, "latencyMs": 3653 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00454, "billedUsd": 0.00454, "estimatedUsd": 0.00454, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 3537 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00176, "billedUsd": 0.00176, "estimatedUsd": 0.00176, "usage": { "prompt_tokens": 116, "completion_tokens": 12, "total_tokens": 128 }, "latencyMs": 3788 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.01013, "billedUsd": 0.01013, "estimatedUsd": 0.01013, "usage": { "prompt_tokens": 108, "completion_tokens": 181, "total_tokens": 289 }, "latencyMs": 5680 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00143, "billedUsd": 0.00143, "estimatedUsd": 0.00143, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 3535 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00424, "billedUsd": 0.00424, "estimatedUsd": 0.00424, "usage": { "prompt_tokens": 124, "completion_tokens": 60, "total_tokens": 184 }, "latencyMs": 4750 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00132, "billedUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 3346 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected fetch" }, "output": "", "spendUsd": 0, "billedUsd": 0, "estimatedUsd": 0, "usage": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "latencyMs": 1473 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00265, "billedUsd": 0.00265, "estimatedUsd": 0.00265, "usage": { "prompt_tokens": 85, "completion_tokens": 36, "total_tokens": 121 }, "latencyMs": 4649 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00091, "billedUsd": 0.00091, "estimatedUsd": 0.00091, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 2887 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0029500000000000004, "billedUsd": 0.00295, "estimatedUsd": 0.0029500000000000004, "usage": { "prompt_tokens": 70, "completion_tokens": 45, "total_tokens": 115 }, "latencyMs": 5896 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 76, "completion_tokens": 12, "total_tokens": 88 }, "latencyMs": 4521 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00141, "billedUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 2960 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0010600000000000002, "billedUsd": 0.00106, "estimatedUsd": 0.0010600000000000002, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 3556 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0011, "billedUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 3052 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00132, "billedUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 3216 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0033299999999999996, "billedUsd": 0.00333, "estimatedUsd": 0.0033299999999999996, "usage": { "prompt_tokens": 98, "completion_tokens": 47, "total_tokens": 145 }, "latencyMs": 4146 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0025599999999999998, "billedUsd": 0.00256, "estimatedUsd": 0.0025599999999999998, "usage": { "prompt_tokens": 96, "completion_tokens": 32, "total_tokens": 128 }, "latencyMs": 5622 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0016100000000000003, "billedUsd": 0.00161, "estimatedUsd": 0.0016100000000000003, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 3304 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00312, "billedUsd": 0.00312, "estimatedUsd": 0.00312, "usage": { "prompt_tokens": 102, "completion_tokens": 42, "total_tokens": 144 }, "latencyMs": 5749 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0012100000000000001, "billedUsd": 0.00121, "estimatedUsd": 0.0012100000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 2937 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.00493, "billedUsd": 0.00493, "estimatedUsd": 0.00493, "usage": { "prompt_tokens": 123, "completion_tokens": 74, "total_tokens": 197 }, "latencyMs": 6598 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0011099999999999999, "billedUsd": 0.00111, "estimatedUsd": 0.0011099999999999999, "usage": { "prompt_tokens": 91, "completion_tokens": 4, "total_tokens": 95 }, "latencyMs": 3360 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0010899999999999998, "billedUsd": 0.00109, "estimatedUsd": 0.0010899999999999998, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 3350 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00104, "billedUsd": 0.00104, "estimatedUsd": 0.00104, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 3860 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0014399999999999999, "billedUsd": 0.00144, "estimatedUsd": 0.0014399999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 9, "total_tokens": 108 }, "latencyMs": 3027 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001353, "billedUsd": 0.00128535, "estimatedUsd": 0.001353, "usage": { "prompt_tokens": 161, "completion_tokens": 58, "total_tokens": 219 }, "latencyMs": 3888 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.002376, "billedUsd": 0.0020592, "estimatedUsd": 0.002376, "usage": { "prompt_tokens": 212, "completion_tokens": 116, "total_tokens": 328 }, "latencyMs": 1084 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.001047, "billedUsd": 0.001047, "estimatedUsd": 0.001047, "usage": { "prompt_tokens": 149, "completion_tokens": 40, "total_tokens": 189 }, "latencyMs": 2305 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001239, "billedUsd": 0.001239, "estimatedUsd": 0.001239, "usage": { "prompt_tokens": 148, "completion_tokens": 53, "total_tokens": 201 }, "latencyMs": 2426 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.001647, "billedUsd": 0.00156465, "estimatedUsd": 0.001647, "usage": { "prompt_tokens": 164, "completion_tokens": 77, "total_tokens": 241 }, "latencyMs": 2966 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001251, "billedUsd": 0.001251, "estimatedUsd": 0.001251, "usage": { "prompt_tokens": 192, "completion_tokens": 45, "total_tokens": 237 }, "latencyMs": 7135 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000882, "billedUsd": 0.000882, "estimatedUsd": 0.000882, "usage": { "prompt_tokens": 154, "completion_tokens": 28, "total_tokens": 182 }, "latencyMs": 2886 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.001461, "billedUsd": 0.001461, "estimatedUsd": 0.001461, "usage": { "prompt_tokens": 192, "completion_tokens": 59, "total_tokens": 251 }, "latencyMs": 3912 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.002643, "billedUsd": 0.00251085, "estimatedUsd": 0.002643, "usage": { "prompt_tokens": 186, "completion_tokens": 139, "total_tokens": 325 }, "latencyMs": 10492 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.002622, "billedUsd": 0.002622, "estimatedUsd": 0.002622, "usage": { "prompt_tokens": 169, "completion_tokens": 141, "total_tokens": 310 }, "latencyMs": 5137 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.001533, "billedUsd": 0.0013986, "estimatedUsd": 0.001533, "usage": { "prompt_tokens": 161, "completion_tokens": 70, "total_tokens": 231 }, "latencyMs": 11339 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.002577, "billedUsd": 0.0024042, "estimatedUsd": 0.002577, "usage": { "prompt_tokens": 169, "completion_tokens": 138, "total_tokens": 307 }, "latencyMs": 1402 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.002733, "billedUsd": 0.002733, "estimatedUsd": 0.002733, "usage": { "prompt_tokens": 206, "completion_tokens": 141, "total_tokens": 347 }, "latencyMs": 7933 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.002043, "billedUsd": 0.002043, "estimatedUsd": 0.002043, "usage": { "prompt_tokens": 156, "completion_tokens": 105, "total_tokens": 261 }, "latencyMs": 3205 }, { "taskId": "classify-001", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00201, "billedUsd": 0.00201, "estimatedUsd": 0.00201, "usage": { "prompt_tokens": 195, "completion_tokens": 95, "total_tokens": 290 }, "latencyMs": 5863 }, { "taskId": "classify-002", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.001911, "billedUsd": 0.001911, "estimatedUsd": 0.001911, "usage": { "prompt_tokens": 192, "completion_tokens": 89, "total_tokens": 281 }, "latencyMs": 8944 }, { "taskId": "classify-003", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0019529999999999999, "billedUsd": 0.0016926, "estimatedUsd": 0.0019529999999999999, "usage": { "prompt_tokens": 191, "completion_tokens": 92, "total_tokens": 283 }, "latencyMs": 1275 }, { "taskId": "classify-004", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.001938, "billedUsd": 0.0016796, "estimatedUsd": 0.001938, "usage": { "prompt_tokens": 191, "completion_tokens": 91, "total_tokens": 282 }, "latencyMs": 1091 }, { "taskId": "classify-005", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.002085, "billedUsd": 0.002085, "estimatedUsd": 0.002085, "usage": { "prompt_tokens": 185, "completion_tokens": 102, "total_tokens": 287 }, "latencyMs": 5268 }, { "taskId": "classify-006", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0018750000000000001, "billedUsd": 0.001875, "estimatedUsd": 0.0018750000000000001, "usage": { "prompt_tokens": 145, "completion_tokens": 96, "total_tokens": 241 }, "latencyMs": 7192 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded but failed multi-step arithmetic reasoning", "spendUsd": 0.002859, "billedUsd": 0.002859, "estimatedUsd": 0.002859, "usage": { "prompt_tokens": 173, "completion_tokens": 156, "total_tokens": 329 }, "latencyMs": 7608 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0015660000000000001, "billedUsd": 0.0013572, "estimatedUsd": 0.0015660000000000001, "usage": { "prompt_tokens": 217, "completion_tokens": 61, "total_tokens": 278 }, "latencyMs": 902 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "High verbosity raises cost, hurting dollars per correct answer.", "spendUsd": 0.002589, "billedUsd": 0.0041355, "estimatedUsd": 0.002589, "usage": { "prompt_tokens": 168, "completion_tokens": 139, "total_tokens": 307 }, "latencyMs": 21808 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.00279, "billedUsd": 0.00279, "estimatedUsd": 0.00279, "usage": { "prompt_tokens": 160, "completion_tokens": 154, "total_tokens": 314 }, "latencyMs": 8042 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0045839999999999995, "billedUsd": 0.004584, "estimatedUsd": 0.0045839999999999995, "usage": { "prompt_tokens": 173, "completion_tokens": 271, "total_tokens": 444 }, "latencyMs": 8258 }, { "taskId": "code-001", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.002019, "billedUsd": 0.002019, "estimatedUsd": 0.002019, "usage": { "prompt_tokens": 163, "completion_tokens": 102, "total_tokens": 265 }, "latencyMs": 3478 }, { "taskId": "code-002", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 0.85" }, "output": "", "spendUsd": 0.015854999999999998, "billedUsd": 0.0156822, "estimatedUsd": 0.015854999999999998, "usage": { "prompt_tokens": 165, "completion_tokens": 1024, "total_tokens": 1189 }, "latencyMs": 28405 }, { "taskId": "code-003", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.001977, "billedUsd": 0.001977, "estimatedUsd": 0.001977, "usage": { "prompt_tokens": 194, "completion_tokens": 93, "total_tokens": 287 }, "latencyMs": 4580 }, { "taskId": "code-004", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.003207, "billedUsd": 0.0027794, "estimatedUsd": 0.003207, "usage": { "prompt_tokens": 204, "completion_tokens": 173, "total_tokens": 377 }, "latencyMs": 1346 }, { "taskId": "code-005", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.003072, "billedUsd": 0.003072, "estimatedUsd": 0.003072, "usage": { "prompt_tokens": 149, "completion_tokens": 175, "total_tokens": 324 }, "latencyMs": 4844 }, { "taskId": "code-006", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0014130000000000002, "billedUsd": 0.0012246, "estimatedUsd": 0.0014130000000000002, "usage": { "prompt_tokens": 191, "completion_tokens": 56, "total_tokens": 247 }, "latencyMs": 939 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.001434, "billedUsd": 0.001434, "estimatedUsd": 0.001434, "usage": { "prompt_tokens": 178, "completion_tokens": 60, "total_tokens": 238 }, "latencyMs": 2760 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.001419, "billedUsd": 0.001419, "estimatedUsd": 0.001419, "usage": { "prompt_tokens": 178, "completion_tokens": 59, "total_tokens": 237 }, "latencyMs": 24276 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001644, "billedUsd": 0.001644, "estimatedUsd": 0.001644, "usage": { "prompt_tokens": 133, "completion_tokens": 83, "total_tokens": 216 }, "latencyMs": 4375 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.001191, "billedUsd": 0.0010322, "estimatedUsd": 0.001191, "usage": { "prompt_tokens": 172, "completion_tokens": 45, "total_tokens": 217 }, "latencyMs": 2035 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0016049999999999999, "billedUsd": 0.001605, "estimatedUsd": 0.0016049999999999999, "usage": { "prompt_tokens": 190, "completion_tokens": 69, "total_tokens": 259 }, "latencyMs": 3311 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.002511, "billedUsd": 0.002511, "estimatedUsd": 0.002511, "usage": { "prompt_tokens": 142, "completion_tokens": 139, "total_tokens": 281 }, "latencyMs": 4410 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0017850000000000001, "billedUsd": 0.001785, "estimatedUsd": 0.0017850000000000001, "usage": { "prompt_tokens": 180, "completion_tokens": 83, "total_tokens": 263 }, "latencyMs": 6059 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00156, "billedUsd": 0.001352, "estimatedUsd": 0.00156, "usage": { "prompt_tokens": 190, "completion_tokens": 66, "total_tokens": 256 }, "latencyMs": 960 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.002082, "billedUsd": 0.0017697, "estimatedUsd": 0.002082, "usage": { "prompt_tokens": 154, "completion_tokens": 108, "total_tokens": 262 }, "latencyMs": 5488 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001557, "billedUsd": 0.00123345, "estimatedUsd": 0.001557, "usage": { "prompt_tokens": 184, "completion_tokens": 67, "total_tokens": 251 }, "latencyMs": 1345 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.002763, "billedUsd": 0.0023946, "estimatedUsd": 0.002763, "usage": { "prompt_tokens": 216, "completion_tokens": 141, "total_tokens": 357 }, "latencyMs": 1642 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.001329, "billedUsd": 0.0011562, "estimatedUsd": 0.001329, "usage": { "prompt_tokens": 158, "completion_tokens": 57, "total_tokens": 215 }, "latencyMs": 4432 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.001455, "billedUsd": 0.00114945, "estimatedUsd": 0.001455, "usage": { "prompt_tokens": 190, "completion_tokens": 59, "total_tokens": 249 }, "latencyMs": 1592 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0023610000000000003, "billedUsd": 0.0020462, "estimatedUsd": 0.0023610000000000003, "usage": { "prompt_tokens": 202, "completion_tokens": 117, "total_tokens": 319 }, "latencyMs": 1284 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001956, "billedUsd": 0.0016952, "estimatedUsd": 0.001956, "usage": { "prompt_tokens": 187, "completion_tokens": 93, "total_tokens": 280 }, "latencyMs": 1174 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001503, "billedUsd": 0.00115476, "estimatedUsd": 0.001503, "usage": { "prompt_tokens": 181, "completion_tokens": 64, "total_tokens": 245 }, "latencyMs": 1925 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.001422, "billedUsd": 0.0012324, "estimatedUsd": 0.001422, "usage": { "prompt_tokens": 184, "completion_tokens": 58, "total_tokens": 242 }, "latencyMs": 869 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.002247, "billedUsd": 0.0018222, "estimatedUsd": 0.002247, "usage": { "prompt_tokens": 189, "completion_tokens": 112, "total_tokens": 301 }, "latencyMs": 3049 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000605, "billedUsd": 0.000605, "estimatedUsd": 0.000605, "usage": { "prompt_tokens": 106, "completion_tokens": 3, "total_tokens": 109 }, "latencyMs": 1365 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0007, "billedUsd": 0.0007, "estimatedUsd": 0.0007, "usage": { "prompt_tokens": 125, "completion_tokens": 3, "total_tokens": 128 }, "latencyMs": 979 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 1316 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0005250000000000001, "billedUsd": 0.000525, "estimatedUsd": 0.0005250000000000001, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 1048 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000625, "billedUsd": 0.000625, "estimatedUsd": 0.000625, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 993 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000615, "billedUsd": 0.000615, "estimatedUsd": 0.000615, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 998 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00062, "billedUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 1065 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 1166 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.001895, "billedUsd": 0.001895, "estimatedUsd": 0.001895, "usage": { "prompt_tokens": 144, "completion_tokens": 47, "total_tokens": 191 }, "latencyMs": 1533 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.001785, "billedUsd": 0.001785, "estimatedUsd": 0.001785, "usage": { "prompt_tokens": 127, "completion_tokens": 46, "total_tokens": 173 }, "latencyMs": 1579 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000705, "billedUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 1112 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0017850000000000001, "billedUsd": 0.001785, "estimatedUsd": 0.0017850000000000001, "usage": { "prompt_tokens": 132, "completion_tokens": 45, "total_tokens": 177 }, "latencyMs": 1777 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0014100000000000002, "billedUsd": 0.00141, "estimatedUsd": 0.0014100000000000002, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 1850 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00146, "billedUsd": 0.00146, "estimatedUsd": 0.00146, "usage": { "prompt_tokens": 107, "completion_tokens": 37, "total_tokens": 144 }, "latencyMs": 1258 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 907 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1009 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0006199999999999999, "billedUsd": 0.00062, "estimatedUsd": 0.0006199999999999999, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 2066 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 988 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000575, "billedUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 1341 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000535, "billedUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 1161 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": false, "grade": { "correct": false, "missing": [], "constraintFailures": [ "wordCount=7, expected=8" ], "wordCount": 7, "sentenceCount": 1 }, "output": "Cheaper model handled extraction, failed multi-step arithmetic.", "spendUsd": 0.0013999999999999998, "billedUsd": 0.0014, "estimatedUsd": 0.0013999999999999998, "usage": { "prompt_tokens": 150, "completion_tokens": 26, "total_tokens": 176 }, "latencyMs": 2946 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0013, "billedUsd": 0.0013, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 1109 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "Verbosity increases token count, driving up cost per answer.", "spendUsd": 0.00115, "billedUsd": 0.00115, "estimatedUsd": 0.00115, "usage": { "prompt_tokens": 125, "completion_tokens": 21, "total_tokens": 146 }, "latencyMs": 1740 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.000895, "billedUsd": 0.000895, "estimatedUsd": 0.000895, "usage": { "prompt_tokens": 114, "completion_tokens": 13, "total_tokens": 127 }, "latencyMs": 1710 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00227, "billedUsd": 0.00227, "estimatedUsd": 0.00227, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 1333 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000655, "billedUsd": 0.000655, "estimatedUsd": 0.000655, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 1081 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000665, "billedUsd": 0.000665, "estimatedUsd": 0.000665, "usage": { "prompt_tokens": 108, "completion_tokens": 5, "total_tokens": 113 }, "latencyMs": 1355 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000715, "billedUsd": 0.000715, "estimatedUsd": 0.000715, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 1685 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0014199999999999998, "billedUsd": 0.00142, "estimatedUsd": 0.0014199999999999998, "usage": { "prompt_tokens": 124, "completion_tokens": 32, "total_tokens": 156 }, "latencyMs": 1334 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 1137 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00062, "billedUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 1187 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 1230 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000455, "billedUsd": 0.000455, "estimatedUsd": 0.000455, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 1178 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000575, "billedUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 1158 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0005300000000000001, "billedUsd": 0.00053, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 1787 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000705, "billedUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 1361 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0005300000000000001, "billedUsd": 0.00053, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 1260 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 1338 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 1001 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000815, "billedUsd": 0.000815, "estimatedUsd": 0.000815, "usage": { "prompt_tokens": 98, "completion_tokens": 13, "total_tokens": 111 }, "latencyMs": 1573 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000555, "billedUsd": 0.000555, "estimatedUsd": 0.000555, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 2164 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0008050000000000002, "billedUsd": 0.000805, "estimatedUsd": 0.0008050000000000002, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 1163 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000585, "billedUsd": 0.000585, "estimatedUsd": 0.000585, "usage": { "prompt_tokens": 102, "completion_tokens": 3, "total_tokens": 105 }, "latencyMs": 969 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006050000000000001, "billedUsd": 0.000605, "estimatedUsd": 0.0006050000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 1237 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.00169, "billedUsd": 0.00169, "estimatedUsd": 0.00169, "usage": { "prompt_tokens": 123, "completion_tokens": 43, "total_tokens": 166 }, "latencyMs": 2227 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00058, "billedUsd": 0.00058, "estimatedUsd": 0.00058, "usage": { "prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96 }, "latencyMs": 1139 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0005449999999999999, "billedUsd": 0.000545, "estimatedUsd": 0.0005449999999999999, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 1103 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00052, "billedUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 984 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0007199999999999999, "billedUsd": 0.00072, "estimatedUsd": 0.0007199999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 9, "total_tokens": 108 }, "latencyMs": 1812 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00108, "billedUsd": 0.00108, "estimatedUsd": 0.00108, "usage": { "prompt_tokens": 106, "completion_tokens": 22, "total_tokens": 128 }, "latencyMs": 3646 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0025, "billedUsd": 0.0025, "estimatedUsd": 0.0025, "usage": { "prompt_tokens": 125, "completion_tokens": 75, "total_tokens": 200 }, "latencyMs": 3680 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000975, "billedUsd": 0.000975, "estimatedUsd": 0.000975, "usage": { "prompt_tokens": 90, "completion_tokens": 21, "total_tokens": 111 }, "latencyMs": 4324 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0006000000000000001, "billedUsd": 0.0006, "estimatedUsd": 0.0006000000000000001, "usage": { "prompt_tokens": 90, "completion_tokens": 6, "total_tokens": 96 }, "latencyMs": 3961 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.001425, "billedUsd": 0.001425, "estimatedUsd": 0.001425, "usage": { "prompt_tokens": 110, "completion_tokens": 35, "total_tokens": 145 }, "latencyMs": 3898 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00099, "billedUsd": 0.00099, "estimatedUsd": 0.00099, "usage": { "prompt_tokens": 108, "completion_tokens": 18, "total_tokens": 126 }, "latencyMs": 4231 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00087, "billedUsd": 0.00087, "estimatedUsd": 0.00087, "usage": { "prompt_tokens": 109, "completion_tokens": 13, "total_tokens": 122 }, "latencyMs": 3551 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 2050 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.002245, "billedUsd": 0.002245, "estimatedUsd": 0.002245, "usage": { "prompt_tokens": 144, "completion_tokens": 61, "total_tokens": 205 }, "latencyMs": 3861 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00211, "billedUsd": 0.00211, "estimatedUsd": 0.00211, "usage": { "prompt_tokens": 127, "completion_tokens": 59, "total_tokens": 186 }, "latencyMs": 5414 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00118, "billedUsd": 0.00118, "estimatedUsd": 0.00118, "usage": { "prompt_tokens": 116, "completion_tokens": 24, "total_tokens": 140 }, "latencyMs": 4376 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0022849999999999997, "billedUsd": 0.002285, "estimatedUsd": 0.0022849999999999997, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 2988 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00241, "billedUsd": 0.00241, "estimatedUsd": 0.00241, "usage": { "prompt_tokens": 117, "completion_tokens": 73, "total_tokens": 190 }, "latencyMs": 4103 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00146, "billedUsd": 0.00146, "estimatedUsd": 0.00146, "usage": { "prompt_tokens": 107, "completion_tokens": 37, "total_tokens": 144 }, "latencyMs": 2652 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 1973 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1924 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.001045, "billedUsd": 0.001045, "estimatedUsd": 0.001045, "usage": { "prompt_tokens": 104, "completion_tokens": 21, "total_tokens": 125 }, "latencyMs": 4491 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 2013 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0021, "billedUsd": 0.0021, "estimatedUsd": 0.0021, "usage": { "prompt_tokens": 95, "completion_tokens": 65, "total_tokens": 160 }, "latencyMs": 4189 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0013349999999999998, "billedUsd": 0.001335, "estimatedUsd": 0.0013349999999999998, "usage": { "prompt_tokens": 92, "completion_tokens": 35, "total_tokens": 127 }, "latencyMs": 4088 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model nailed extraction, failed arithmetic, suits parsing", "spendUsd": 0.014199999999999999, "billedUsd": 0.0142, "estimatedUsd": 0.014199999999999999, "usage": { "prompt_tokens": 150, "completion_tokens": 538, "total_tokens": 688 }, "latencyMs": 8238 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0013, "billedUsd": 0.0013, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 3733 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity inflates cost per correct answer, undermining accuracy gains.", "spendUsd": 0.0028250000000000003, "billedUsd": 0.002825, "estimatedUsd": 0.0028250000000000003, "usage": { "prompt_tokens": 125, "completion_tokens": 88, "total_tokens": 213 }, "latencyMs": 5301 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The Cost of Each Correct Answer", "spendUsd": 0.002345, "billedUsd": 0.002345, "estimatedUsd": 0.002345, "usage": { "prompt_tokens": 114, "completion_tokens": 71, "total_tokens": 185 }, "latencyMs": 6285 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 27, "sentenceCount": 1 }, "output": "OpenRouter handles the routing, while costs are derived from AI Pricing Guru's direct-provider list prices so the metric mirrors what API buyers see on provider pricing pages.", "spendUsd": 0.00337, "billedUsd": 0.00337, "estimatedUsd": 0.00337, "usage": { "prompt_tokens": 149, "completion_tokens": 105, "total_tokens": 254 }, "latencyMs": 5600 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00108, "billedUsd": 0.00108, "estimatedUsd": 0.00108, "usage": { "prompt_tokens": 116, "completion_tokens": 20, "total_tokens": 136 }, "latencyMs": 5924 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0023150000000000002, "billedUsd": 0.002315, "estimatedUsd": 0.0023150000000000002, "usage": { "prompt_tokens": 108, "completion_tokens": 71, "total_tokens": 179 }, "latencyMs": 4902 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "`inputPerM`", "spendUsd": 0.001115, "billedUsd": 0.001115, "estimatedUsd": 0.001115, "usage": { "prompt_tokens": 108, "completion_tokens": 23, "total_tokens": 131 }, "latencyMs": 4029 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.00357, "billedUsd": 0.00357, "estimatedUsd": 0.00357, "usage": { "prompt_tokens": 124, "completion_tokens": 118, "total_tokens": 242 }, "latencyMs": 5601 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.001535, "billedUsd": 0.001535, "estimatedUsd": 0.001535, "usage": { "prompt_tokens": 97, "completion_tokens": 42, "total_tokens": 139 }, "latencyMs": 4880 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000995, "billedUsd": 0.000995, "estimatedUsd": 0.000995, "usage": { "prompt_tokens": 99, "completion_tokens": 20, "total_tokens": 119 }, "latencyMs": 4811 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0014, "billedUsd": 0.0014, "estimatedUsd": 0.0014, "usage": { "prompt_tokens": 85, "completion_tokens": 39, "total_tokens": 124 }, "latencyMs": 3739 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00078, "billedUsd": 0.00078, "estimatedUsd": 0.00078, "usage": { "prompt_tokens": 76, "completion_tokens": 16, "total_tokens": 92 }, "latencyMs": 3980 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0021, "billedUsd": 0.0021, "estimatedUsd": 0.0021, "usage": { "prompt_tokens": 70, "completion_tokens": 70, "total_tokens": 140 }, "latencyMs": 5733 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0012799999999999999, "billedUsd": 0.00128, "estimatedUsd": 0.0012799999999999999, "usage": { "prompt_tokens": 76, "completion_tokens": 36, "total_tokens": 112 }, "latencyMs": 3947 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00113, "billedUsd": 0.00113, "estimatedUsd": 0.00113, "usage": { "prompt_tokens": 101, "completion_tokens": 25, "total_tokens": 126 }, "latencyMs": 5959 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.001305, "billedUsd": 0.001305, "estimatedUsd": 0.001305, "usage": { "prompt_tokens": 86, "completion_tokens": 35, "total_tokens": 121 }, "latencyMs": 4714 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.002225, "billedUsd": 0.002225, "estimatedUsd": 0.002225, "usage": { "prompt_tokens": 85, "completion_tokens": 72, "total_tokens": 157 }, "latencyMs": 5619 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 2465 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.002615, "billedUsd": 0.002615, "estimatedUsd": 0.002615, "usage": { "prompt_tokens": 98, "completion_tokens": 85, "total_tokens": 183 }, "latencyMs": 5531 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0012799999999999999, "billedUsd": 0.00128, "estimatedUsd": 0.0012799999999999999, "usage": { "prompt_tokens": 96, "completion_tokens": 32, "total_tokens": 128 }, "latencyMs": 6169 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0008050000000000002, "billedUsd": 0.000805, "estimatedUsd": 0.0008050000000000002, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 2110 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.001935, "billedUsd": 0.001935, "estimatedUsd": 0.001935, "usage": { "prompt_tokens": 102, "completion_tokens": 57, "total_tokens": 159 }, "latencyMs": 5472 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006050000000000001, "billedUsd": 0.000605, "estimatedUsd": 0.0006050000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 2042 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.00309, "billedUsd": 0.00309, "estimatedUsd": 0.00309, "usage": { "prompt_tokens": 123, "completion_tokens": 99, "total_tokens": 222 }, "latencyMs": 3844 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.001955, "billedUsd": 0.001955, "estimatedUsd": 0.001955, "usage": { "prompt_tokens": 91, "completion_tokens": 60, "total_tokens": 151 }, "latencyMs": 5557 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001195, "billedUsd": 0.001195, "estimatedUsd": 0.001195, "usage": { "prompt_tokens": 84, "completion_tokens": 31, "total_tokens": 115 }, "latencyMs": 5414 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00112, "billedUsd": 0.00112, "estimatedUsd": 0.00112, "usage": { "prompt_tokens": 89, "completion_tokens": 27, "total_tokens": 116 }, "latencyMs": 3977 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0018449999999999999, "billedUsd": 0.001845, "estimatedUsd": 0.0018449999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 54, "total_tokens": 153 }, "latencyMs": 5850 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000472, "billedUsd": 0.000472, "estimatedUsd": 0.000472, "usage": { "prompt_tokens": 106, "completion_tokens": 26, "total_tokens": 132 }, "latencyMs": 5708 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00102, "billedUsd": 0.00102, "estimatedUsd": 0.00102, "usage": { "prompt_tokens": 125, "completion_tokens": 77, "total_tokens": 202 }, "latencyMs": 4729 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00022, "billedUsd": 0.00022, "estimatedUsd": 0.00022, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 2060 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00021, "billedUsd": 0.00021, "estimatedUsd": 0.00021, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 2040 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00025, "billedUsd": 0.00025, "estimatedUsd": 0.00025, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 2966 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000246, "billedUsd": 0.000246, "estimatedUsd": 0.000246, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 2147 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000248, "billedUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 2085 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000258, "billedUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 2306 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.000758, "billedUsd": 0.000758, "estimatedUsd": 0.000758, "usage": { "prompt_tokens": 144, "completion_tokens": 47, "total_tokens": 191 }, "latencyMs": 2388 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0008939999999999999, "billedUsd": 0.000894, "estimatedUsd": 0.0008939999999999999, "usage": { "prompt_tokens": 127, "completion_tokens": 64, "total_tokens": 191 }, "latencyMs": 2796 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000282, "billedUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 2091 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.000914, "billedUsd": 0.000914, "estimatedUsd": 0.000914, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 2842 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.000564, "billedUsd": 0.000564, "estimatedUsd": 0.000564, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 2278 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.000704, "billedUsd": 0.000704, "estimatedUsd": 0.000704, "usage": { "prompt_tokens": 107, "completion_tokens": 49, "total_tokens": 156 }, "latencyMs": 2389 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000272, "billedUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 2360 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000258, "billedUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 4189 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00024799999999999996, "billedUsd": 0.000248, "estimatedUsd": 0.00024799999999999996, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 2247 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000272, "billedUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 2011 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 2064 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000214, "billedUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 3764 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded extraction but failed arithmetic reasoning.", "spendUsd": 0.00058, "billedUsd": 0.00058, "estimatedUsd": 0.00058, "usage": { "prompt_tokens": 150, "completion_tokens": 28, "total_tokens": 178 }, "latencyMs": 2290 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00052, "billedUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 2123 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "High verbosity increases cost, reducing accuracy's practical value.", "spendUsd": 0.00049, "billedUsd": 0.00049, "estimatedUsd": 0.00049, "usage": { "prompt_tokens": 125, "completion_tokens": 24, "total_tokens": 149 }, "latencyMs": 2416 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost per Correct Answer: Model Comparison", "spendUsd": 0.000418, "billedUsd": 0.000418, "estimatedUsd": 0.000418, "usage": { "prompt_tokens": 114, "completion_tokens": 19, "total_tokens": 133 }, "latencyMs": 2238 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000908, "billedUsd": 0.000908, "estimatedUsd": 0.000908, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 2720 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000262, "billedUsd": 0.000262, "estimatedUsd": 0.000262, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 2056 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0006559999999999999, "billedUsd": 0.000656, "estimatedUsd": 0.0006559999999999999, "usage": { "prompt_tokens": 108, "completion_tokens": 44, "total_tokens": 152 }, "latencyMs": 3999 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000286, "billedUsd": 0.000286, "estimatedUsd": 0.000286, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 2543 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001108, "billedUsd": 0.001108, "estimatedUsd": 0.001108, "usage": { "prompt_tokens": 124, "completion_tokens": 86, "total_tokens": 210 }, "latencyMs": 5510 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000264, "billedUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 2033 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000248, "billedUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 2446 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00057, "billedUsd": 0.00057, "estimatedUsd": 0.00057, "usage": { "prompt_tokens": 85, "completion_tokens": 40, "total_tokens": 125 }, "latencyMs": 5566 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 2061 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 2181 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000212, "billedUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 2684 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000282, "billedUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 1916 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000212, "billedUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 2040 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00067, "billedUsd": 0.00067, "estimatedUsd": 0.00067, "usage": { "prompt_tokens": 85, "completion_tokens": 50, "total_tokens": 135 }, "latencyMs": 5118 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000264, "billedUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 2570 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000326, "billedUsd": 0.000326, "estimatedUsd": 0.000326, "usage": { "prompt_tokens": 98, "completion_tokens": 13, "total_tokens": 111 }, "latencyMs": 2449 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000222, "billedUsd": 0.000222, "estimatedUsd": 0.000222, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 2017 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000322, "billedUsd": 0.000322, "estimatedUsd": 0.000322, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 2139 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000684, "billedUsd": 0.000684, "estimatedUsd": 0.000684, "usage": { "prompt_tokens": 102, "completion_tokens": 48, "total_tokens": 150 }, "latencyMs": 4909 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000242, "billedUsd": 0.000242, "estimatedUsd": 0.000242, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 2674 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001206, "billedUsd": 0.001206, "estimatedUsd": 0.001206, "usage": { "prompt_tokens": 123, "completion_tokens": 96, "total_tokens": 219 }, "latencyMs": 4648 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.000232, "billedUsd": 0.000232, "estimatedUsd": 0.000232, "usage": { "prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96 }, "latencyMs": 3843 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00021799999999999999, "billedUsd": 0.000218, "estimatedUsd": 0.00021799999999999999, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 2202 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000208, "billedUsd": 0.000208, "estimatedUsd": 0.000208, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 2222 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "Cost per correct answer", "spendUsd": 0.000298, "billedUsd": 0.000298, "estimatedUsd": 0.000298, "usage": { "prompt_tokens": 99, "completion_tokens": 10, "total_tokens": 109 }, "latencyMs": 2220 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000099, "billedUsd": 0.000099, "estimatedUsd": 0.000099, "usage": { "prompt_tokens": 74, "completion_tokens": 5, "total_tokens": 79 }, "latencyMs": 775 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 6, "detail": "expected 8" }, "output": "6", "spendUsd": 0.000115, "billedUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 90, "completion_tokens": 5, "total_tokens": 95 }, "latencyMs": 814 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 62, "completion_tokens": 6, "total_tokens": 68 }, "latencyMs": 768 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000087, "billedUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 861 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 18, "detail": "expected 20" }, "output": "18", "spendUsd": 0.00009999999999999999, "billedUsd": 0.0001, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 785 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00009499999999999999, "billedUsd": 0.000095, "estimatedUsd": 0.00009499999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 5, "total_tokens": 75 }, "latencyMs": 857 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000093, "billedUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 957 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000104, "billedUsd": 0.000104, "estimatedUsd": 0.000104, "usage": { "prompt_tokens": 69, "completion_tokens": 7, "total_tokens": 76 }, "latencyMs": 979 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.000375, "billedUsd": 0.000375, "estimatedUsd": 0.000375, "usage": { "prompt_tokens": 100, "completion_tokens": 55, "total_tokens": 155 }, "latencyMs": 906 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.000371, "billedUsd": 0.000371, "estimatedUsd": 0.000371, "usage": { "prompt_tokens": 86, "completion_tokens": 57, "total_tokens": 143 }, "latencyMs": 990 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000094, "billedUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 808 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.00038700000000000003, "billedUsd": 0.000387, "estimatedUsd": 0.00038700000000000003, "usage": { "prompt_tokens": 92, "completion_tokens": 59, "total_tokens": 151 }, "latencyMs": 1067 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00021700000000000002, "billedUsd": 0.000217, "estimatedUsd": 0.00021700000000000002, "usage": { "prompt_tokens": 82, "completion_tokens": 27, "total_tokens": 109 }, "latencyMs": 1918 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.00029299999999999997, "billedUsd": 0.000293, "estimatedUsd": 0.00029299999999999997, "usage": { "prompt_tokens": 73, "completion_tokens": 44, "total_tokens": 117 }, "latencyMs": 882 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000094, "billedUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 1000 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000091, "billedUsd": 0.000091, "estimatedUsd": 0.000091, "usage": { "prompt_tokens": 71, "completion_tokens": 4, "total_tokens": 75 }, "latencyMs": 821 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000093, "billedUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 777 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000089, "billedUsd": 0.000089, "estimatedUsd": 0.000089, "usage": { "prompt_tokens": 69, "completion_tokens": 4, "total_tokens": 73 }, "latencyMs": 750 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00007999999999999999, "billedUsd": 0.00008, "estimatedUsd": 0.00007999999999999999, "usage": { "prompt_tokens": 60, "completion_tokens": 4, "total_tokens": 64 }, "latencyMs": 2614 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000078, "billedUsd": 0.000078, "estimatedUsd": 0.000078, "usage": { "prompt_tokens": 58, "completion_tokens": 4, "total_tokens": 62 }, "latencyMs": 750 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction model failed complex arithmetic reasoning tasks.", "spendUsd": 0.000154, "billedUsd": 0.000154, "estimatedUsd": 0.000154, "usage": { "prompt_tokens": 89, "completion_tokens": 13, "total_tokens": 102 }, "latencyMs": 1037 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000165, "billedUsd": 0.000165, "estimatedUsd": 0.000165, "usage": { "prompt_tokens": 95, "completion_tokens": 14, "total_tokens": 109 }, "latencyMs": 815 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer despite model accuracy.", "spendUsd": 0.000156, "billedUsd": 0.000156, "estimatedUsd": 0.000156, "usage": { "prompt_tokens": 81, "completion_tokens": 15, "total_tokens": 96 }, "latencyMs": 841 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost to Correct AI Benchmark Answers", "spendUsd": 0.000134, "billedUsd": 0.000134, "estimatedUsd": 0.000134, "usage": { "prompt_tokens": 74, "completion_tokens": 12, "total_tokens": 86 }, "latencyMs": 851 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00028399999999999996, "billedUsd": 0.000284, "estimatedUsd": 0.00028399999999999996, "usage": { "prompt_tokens": 89, "completion_tokens": 39, "total_tokens": 128 }, "latencyMs": 831 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000106, "billedUsd": 0.000106, "estimatedUsd": 0.000106, "usage": { "prompt_tokens": 81, "completion_tokens": 5, "total_tokens": 86 }, "latencyMs": 740 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000115, "billedUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 80, "completion_tokens": 7, "total_tokens": 87 }, "latencyMs": 774 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00009999999999999999, "billedUsd": 0.0001, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 6, "total_tokens": 76 }, "latencyMs": 850 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.000299, "billedUsd": 0.000299, "estimatedUsd": 0.000299, "usage": { "prompt_tokens": 84, "completion_tokens": 43, "total_tokens": 127 }, "latencyMs": 1074 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00010899999999999999, "billedUsd": 0.000109, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1487 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000088, "billedUsd": 0.000088, "estimatedUsd": 0.000088, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 820 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.000084, "billedUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 54, "completion_tokens": 6, "total_tokens": 60 }, "latencyMs": 834 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00007199999999999999, "billedUsd": 0.000072, "estimatedUsd": 0.00007199999999999999, "usage": { "prompt_tokens": 52, "completion_tokens": 4, "total_tokens": 56 }, "latencyMs": 747 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000096, "billedUsd": 0.000096, "estimatedUsd": 0.000096, "usage": { "prompt_tokens": 46, "completion_tokens": 10, "total_tokens": 56 }, "latencyMs": 995 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000077, "billedUsd": 0.000077, "estimatedUsd": 0.000077, "usage": { "prompt_tokens": 47, "completion_tokens": 6, "total_tokens": 53 }, "latencyMs": 847 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00010899999999999999, "billedUsd": 0.000109, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 69, "completion_tokens": 8, "total_tokens": 77 }, "latencyMs": 791 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00007500000000000001, "billedUsd": 0.000075, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 796 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00007500000000000001, "billedUsd": 0.000075, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 726 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00011300000000000001, "billedUsd": 0.000113, "estimatedUsd": 0.00011300000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 9, "total_tokens": 77 }, "latencyMs": 736 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00013099999999999999, "billedUsd": 0.000131, "estimatedUsd": 0.00013099999999999999, "usage": { "prompt_tokens": 66, "completion_tokens": 13, "total_tokens": 79 }, "latencyMs": 805 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000087, "billedUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 964 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000116, "billedUsd": 0.000116, "estimatedUsd": 0.000116, "usage": { "prompt_tokens": 96, "completion_tokens": 4, "total_tokens": 100 }, "latencyMs": 820 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 72, "completion_tokens": 4, "total_tokens": 76 }, "latencyMs": 867 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000097, "billedUsd": 0.000097, "estimatedUsd": 0.000097, "usage": { "prompt_tokens": 67, "completion_tokens": 6, "total_tokens": 73 }, "latencyMs": 886 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.000315, "billedUsd": 0.000315, "estimatedUsd": 0.000315, "usage": { "prompt_tokens": 85, "completion_tokens": 46, "total_tokens": 131 }, "latencyMs": 959 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes.", "spendUsd": 0.000086, "billedUsd": 0.000086, "estimatedUsd": 0.000086, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 773 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 786 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000084, "billedUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 770 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000098, "billedUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 63, "completion_tokens": 7, "total_tokens": 70 }, "latencyMs": 2001 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0006855, "billedUsd": 0.0006855, "estimatedUsd": 0.0006855, "usage": { "prompt_tokens": 64, "completion_tokens": 170, "total_tokens": 234 }, "latencyMs": 1914 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00098025, "billedUsd": 0.00098025, "estimatedUsd": 0.00098025, "usage": { "prompt_tokens": 87, "completion_tokens": 244, "total_tokens": 331 }, "latencyMs": 1933 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0007785, "billedUsd": 0.0007785, "estimatedUsd": 0.0007785, "usage": { "prompt_tokens": 58, "completion_tokens": 196, "total_tokens": 254 }, "latencyMs": 1595 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00047925, "billedUsd": 0.00047925, "estimatedUsd": 0.00047925, "usage": { "prompt_tokens": 54, "completion_tokens": 117, "total_tokens": 171 }, "latencyMs": 1404 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0008910000000000001, "billedUsd": 0.000891, "estimatedUsd": 0.0008910000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 224, "total_tokens": 292 }, "latencyMs": 1920 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0005655, "billedUsd": 0.0005655, "estimatedUsd": 0.0005655, "usage": { "prompt_tokens": 59, "completion_tokens": 139, "total_tokens": 198 }, "latencyMs": 1490 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00029625, "billedUsd": 0.00029625, "estimatedUsd": 0.00029625, "usage": { "prompt_tokens": 60, "completion_tokens": 67, "total_tokens": 127 }, "latencyMs": 1173 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00033375, "billedUsd": 0.00033375, "estimatedUsd": 0.00033375, "usage": { "prompt_tokens": 60, "completion_tokens": 77, "total_tokens": 137 }, "latencyMs": 1365 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.3, \"outputPerM\": 2.5, \"context\": 1048576 }", "spendUsd": 0.00182625, "billedUsd": 0.00182625, "estimatedUsd": 0.00182625, "usage": { "prompt_tokens": 95, "completion_tokens": 468, "total_tokens": 563 }, "latencyMs": 3377 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0024795, "billedUsd": 0.0024795, "estimatedUsd": 0.0024795, "usage": { "prompt_tokens": 76, "completion_tokens": 646, "total_tokens": 722 }, "latencyMs": 4310 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0008535, "billedUsd": 0.0008535, "estimatedUsd": 0.0008535, "usage": { "prompt_tokens": 63, "completion_tokens": 215, "total_tokens": 278 }, "latencyMs": 2121 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.00240825, "billedUsd": 0.00240825, "estimatedUsd": 0.00240825, "usage": { "prompt_tokens": 86, "completion_tokens": 625, "total_tokens": 711 }, "latencyMs": 3951 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0022635000000000003, "billedUsd": 0.0022635, "estimatedUsd": 0.0022635000000000003, "usage": { "prompt_tokens": 73, "completion_tokens": 589, "total_tokens": 662 }, "latencyMs": 4135 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0010275, "billedUsd": 0.0010275, "estimatedUsd": 0.0010275, "usage": { "prompt_tokens": 60, "completion_tokens": 262, "total_tokens": 322 }, "latencyMs": 2074 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00098325, "billedUsd": 0.00098325, "estimatedUsd": 0.00098325, "usage": { "prompt_tokens": 61, "completion_tokens": 250, "total_tokens": 311 }, "latencyMs": 2137 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000342, "billedUsd": 0.000342, "estimatedUsd": 0.000342, "usage": { "prompt_tokens": 56, "completion_tokens": 80, "total_tokens": 136 }, "latencyMs": 1360 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.001014, "billedUsd": 0.001014, "estimatedUsd": 0.001014, "usage": { "prompt_tokens": 57, "completion_tokens": 259, "total_tokens": 316 }, "latencyMs": 2575 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00028649999999999997, "billedUsd": 0.0002865, "estimatedUsd": 0.00028649999999999997, "usage": { "prompt_tokens": 57, "completion_tokens": 65, "total_tokens": 122 }, "latencyMs": 1188 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0008032499999999999, "billedUsd": 0.00080325, "estimatedUsd": 0.0008032499999999999, "usage": { "prompt_tokens": 51, "completion_tokens": 204, "total_tokens": 255 }, "latencyMs": 2108 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00061425, "billedUsd": 0.00061425, "estimatedUsd": 0.00061425, "usage": { "prompt_tokens": 49, "completion_tokens": 154, "total_tokens": 203 }, "latencyMs": 1720 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model passed extraction but failed arithmetic.", "spendUsd": 0.00362475, "billedUsd": 0.00362475, "estimatedUsd": 0.00362475, "usage": { "prompt_tokens": 78, "completion_tokens": 951, "total_tokens": 1029 }, "latencyMs": 6228 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00111975, "billedUsd": 0.00111975, "estimatedUsd": 0.00111975, "usage": { "prompt_tokens": 83, "completion_tokens": 282, "total_tokens": 365 }, "latencyMs": 2196 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "High output verbosity increases total cost.", "spendUsd": 0.00293775, "billedUsd": 0.00293775, "estimatedUsd": 0.00293775, "usage": { "prompt_tokens": 72, "completion_tokens": 769, "total_tokens": 841 }, "latencyMs": 5604 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "AI Cost Per Correct Answer", "spendUsd": 0.0020955, "billedUsd": 0.0020955, "estimatedUsd": 0.0020955, "usage": { "prompt_tokens": 64, "completion_tokens": 546, "total_tokens": 610 }, "latencyMs": 2788 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0015555, "billedUsd": 0.0015555, "estimatedUsd": 0.0015555, "usage": { "prompt_tokens": 79, "completion_tokens": 399, "total_tokens": 478 }, "latencyMs": 2786 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00105525, "billedUsd": 0.00105525, "estimatedUsd": 0.00105525, "usage": { "prompt_tokens": 72, "completion_tokens": 267, "total_tokens": 339 }, "latencyMs": 2113 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00072975, "billedUsd": 0.00072975, "estimatedUsd": 0.00072975, "usage": { "prompt_tokens": 68, "completion_tokens": 181, "total_tokens": 249 }, "latencyMs": 1982 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00100725, "billedUsd": 0.00100725, "estimatedUsd": 0.00100725, "usage": { "prompt_tokens": 63, "completion_tokens": 256, "total_tokens": 319 }, "latencyMs": 2481 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03}", "spendUsd": 0.0022170000000000002, "billedUsd": 0.002217, "estimatedUsd": 0.0022170000000000002, "usage": { "prompt_tokens": 76, "completion_tokens": 576, "total_tokens": 652 }, "latencyMs": 3208 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00079875, "billedUsd": 0.00079875, "estimatedUsd": 0.00079875, "usage": { "prompt_tokens": 55, "completion_tokens": 202, "total_tokens": 257 }, "latencyMs": 2649 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0007005000000000001, "billedUsd": 0.0007005, "estimatedUsd": 0.0007005000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 175, "total_tokens": 234 }, "latencyMs": 1670 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0006772500000000001, "billedUsd": 0.00067725, "estimatedUsd": 0.0006772500000000001, "usage": { "prompt_tokens": 43, "completion_tokens": 172, "total_tokens": 215 }, "latencyMs": 1578 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0004980000000000001, "billedUsd": 0.000498, "estimatedUsd": 0.0004980000000000001, "usage": { "prompt_tokens": 44, "completion_tokens": 124, "total_tokens": 168 }, "latencyMs": 1303 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0010252500000000001, "billedUsd": 0.00102525, "estimatedUsd": 0.0010252500000000001, "usage": { "prompt_tokens": 37, "completion_tokens": 266, "total_tokens": 303 }, "latencyMs": 1984 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00055725, "billedUsd": 0.00055725, "estimatedUsd": 0.00055725, "usage": { "prompt_tokens": 38, "completion_tokens": 141, "total_tokens": 179 }, "latencyMs": 1842 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000822, "billedUsd": 0.000822, "estimatedUsd": 0.000822, "usage": { "prompt_tokens": 56, "completion_tokens": 208, "total_tokens": 264 }, "latencyMs": 2055 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0005265000000000001, "billedUsd": 0.0005265, "estimatedUsd": 0.0005265000000000001, "usage": { "prompt_tokens": 47, "completion_tokens": 131, "total_tokens": 178 }, "latencyMs": 1618 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000564, "billedUsd": 0.000564, "estimatedUsd": 0.000564, "usage": { "prompt_tokens": 47, "completion_tokens": 141, "total_tokens": 188 }, "latencyMs": 1564 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0009217499999999999, "billedUsd": 0.00092175, "estimatedUsd": 0.0009217499999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 233, "total_tokens": 297 }, "latencyMs": 1861 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000612, "billedUsd": 0.000612, "estimatedUsd": 0.000612, "usage": { "prompt_tokens": 56, "completion_tokens": 152, "total_tokens": 208 }, "latencyMs": 1622 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00086925, "billedUsd": 0.00086925, "estimatedUsd": 0.00086925, "usage": { "prompt_tokens": 49, "completion_tokens": 222, "total_tokens": 271 }, "latencyMs": 1390 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00109575, "billedUsd": 0.00109575, "estimatedUsd": 0.00109575, "usage": { "prompt_tokens": 91, "completion_tokens": 274, "total_tokens": 365 }, "latencyMs": 2211 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000615, "billedUsd": 0.000615, "estimatedUsd": 0.000615, "usage": { "prompt_tokens": 65, "completion_tokens": 151, "total_tokens": 216 }, "latencyMs": 1554 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00037799999999999997, "billedUsd": 0.000378, "estimatedUsd": 0.00037799999999999997, "usage": { "prompt_tokens": 59, "completion_tokens": 89, "total_tokens": 148 }, "latencyMs": 1403 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.001878, "billedUsd": 0.001878, "estimatedUsd": 0.001878, "usage": { "prompt_tokens": 74, "completion_tokens": 486, "total_tokens": 560 }, "latencyMs": 3033 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00087975, "billedUsd": 0.00087975, "estimatedUsd": 0.00087975, "usage": { "prompt_tokens": 53, "completion_tokens": 224, "total_tokens": 277 }, "latencyMs": 2292 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0006825000000000001, "billedUsd": 0.0006825, "estimatedUsd": 0.0006825000000000001, "usage": { "prompt_tokens": 50, "completion_tokens": 172, "total_tokens": 222 }, "latencyMs": 1626 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0005745, "billedUsd": 0.0005745, "estimatedUsd": 0.0005745, "usage": { "prompt_tokens": 51, "completion_tokens": 143, "total_tokens": 194 }, "latencyMs": 1599 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0009375, "billedUsd": 0.0009375, "estimatedUsd": 0.0009375, "usage": { "prompt_tokens": 55, "completion_tokens": 239, "total_tokens": 294 }, "latencyMs": 2468 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0000267, "billedUsd": 0.0000267, "estimatedUsd": 0.0000267, "usage": { "prompt_tokens": 64, "completion_tokens": 3, "total_tokens": 67 }, "latencyMs": 750 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "actual": null, "detail": "expected 8" }, "output": "To find the total cost in cents: 1. **Input tokens:** - 1M tokens = $0.30 - 100,000 tokens = $0.30 / 10 = $0.03 2. **Output tokens:** - 1M tokens = $2.50 - 20,000 tokens = $2.50 * (20,000 / 1,000,000) = $2.50 * 0.02 = $0.05 3. **Total cost in dollars:** - $0.03 + $0.05 = $0.08 4. **Convert to cents:** - $0.08 * 100 = 8 cents 8", "spendUsd": 0.0005011000000000001, "billedUsd": 0.0005011, "estimatedUsd": 0.0005011000000000001, "usage": { "prompt_tokens": 87, "completion_tokens": 190, "total_tokens": 277 }, "latencyMs": 1101 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000029900000000000002, "billedUsd": 0.0000299, "estimatedUsd": 0.000029900000000000002, "usage": { "prompt_tokens": 58, "completion_tokens": 5, "total_tokens": 63 }, "latencyMs": 507 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000021199999999999997, "billedUsd": 0.0000212, "estimatedUsd": 0.000021199999999999997, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 486 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000025399999999999997, "billedUsd": 0.0000254, "estimatedUsd": 0.000025399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 2, "total_tokens": 70 }, "latencyMs": 455 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0000227, "billedUsd": 0.0000227, "estimatedUsd": 0.0000227, "usage": { "prompt_tokens": 59, "completion_tokens": 2, "total_tokens": 61 }, "latencyMs": 850 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000023, "billedUsd": 0.000023, "estimatedUsd": 0.000023, "usage": { "prompt_tokens": 60, "completion_tokens": 2, "total_tokens": 62 }, "latencyMs": 502 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000028, "billedUsd": 0.000028, "estimatedUsd": 0.000028, "usage": { "prompt_tokens": 60, "completion_tokens": 4, "total_tokens": 64 }, "latencyMs": 676 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.00017099999999999998, "billedUsd": 0.000171, "estimatedUsd": 0.00017099999999999998, "usage": { "prompt_tokens": 95, "completion_tokens": 57, "total_tokens": 152 }, "latencyMs": 898 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0001628, "billedUsd": 0.0001628, "estimatedUsd": 0.0001628, "usage": { "prompt_tokens": 76, "completion_tokens": 56, "total_tokens": 132 }, "latencyMs": 611 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0000214, "billedUsd": 0.0000214, "estimatedUsd": 0.0000214, "usage": { "prompt_tokens": 63, "completion_tokens": 1, "total_tokens": 64 }, "latencyMs": 575 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0001608, "billedUsd": 0.0001608, "estimatedUsd": 0.0001608, "usage": { "prompt_tokens": 86, "completion_tokens": 54, "total_tokens": 140 }, "latencyMs": 593 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0000844, "billedUsd": 0.0000844, "estimatedUsd": 0.0000844, "usage": { "prompt_tokens": 73, "completion_tokens": 25, "total_tokens": 98 }, "latencyMs": 584 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.00010300000000000001, "billedUsd": 0.000103, "estimatedUsd": 0.00010300000000000001, "usage": { "prompt_tokens": 60, "completion_tokens": 34, "total_tokens": 94 }, "latencyMs": 543 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000020799999999999997, "billedUsd": 0.0000208, "estimatedUsd": 0.000020799999999999997, "usage": { "prompt_tokens": 61, "completion_tokens": 1, "total_tokens": 62 }, "latencyMs": 562 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000019299999999999998, "billedUsd": 0.0000193, "estimatedUsd": 0.000019299999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 1, "total_tokens": 57 }, "latencyMs": 566 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000221, "billedUsd": 0.0000221, "estimatedUsd": 0.0000221, "usage": { "prompt_tokens": 57, "completion_tokens": 2, "total_tokens": 59 }, "latencyMs": 700 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0000196, "billedUsd": 0.0000196, "estimatedUsd": 0.0000196, "usage": { "prompt_tokens": 57, "completion_tokens": 1, "total_tokens": 58 }, "latencyMs": 410 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000178, "billedUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 51, "completion_tokens": 1, "total_tokens": 52 }, "latencyMs": 552 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000017199999999999998, "billedUsd": 0.0000172, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 393 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "missing": [], "constraintFailures": [ "wordCount=7, expected=8" ], "wordCount": 7, "sentenceCount": 1 }, "output": "Cheaper model aced extraction, failed multi-step arithmetic.", "spendUsd": 0.0000559, "billedUsd": 0.0000559, "estimatedUsd": 0.0000559, "usage": { "prompt_tokens": 78, "completion_tokens": 13, "total_tokens": 91 }, "latencyMs": 540 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000499, "billedUsd": 0.0000499, "estimatedUsd": 0.0000499, "usage": { "prompt_tokens": 83, "completion_tokens": 10, "total_tokens": 93 }, "latencyMs": 518 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "High verbosity drives up cost despite great accuracy.", "spendUsd": 0.0000466, "billedUsd": 0.0000466, "estimatedUsd": 0.0000466, "usage": { "prompt_tokens": 72, "completion_tokens": 10, "total_tokens": 82 }, "latencyMs": 586 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The Cost of a Correct Answer", "spendUsd": 0.0000342, "billedUsd": 0.0000342, "estimatedUsd": 0.0000342, "usage": { "prompt_tokens": 64, "completion_tokens": 6, "total_tokens": 70 }, "latencyMs": 503 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00011119999999999998, "billedUsd": 0.0001112, "estimatedUsd": 0.00011119999999999998, "usage": { "prompt_tokens": 79, "completion_tokens": 35, "total_tokens": 114 }, "latencyMs": 618 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0000266, "billedUsd": 0.0000266, "estimatedUsd": 0.0000266, "usage": { "prompt_tokens": 72, "completion_tokens": 2, "total_tokens": 74 }, "latencyMs": 417 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000030399999999999997, "billedUsd": 0.0000304, "estimatedUsd": 0.000030399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 501 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000026399999999999998, "billedUsd": 0.0000264, "estimatedUsd": 0.000026399999999999998, "usage": { "prompt_tokens": 63, "completion_tokens": 3, "total_tokens": 66 }, "latencyMs": 605 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.00011029999999999998, "billedUsd": 0.0001103, "estimatedUsd": 0.00011029999999999998, "usage": { "prompt_tokens": 76, "completion_tokens": 35, "total_tokens": 111 }, "latencyMs": 598 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000029, "billedUsd": 0.000029, "estimatedUsd": 0.000029, "usage": { "prompt_tokens": 55, "completion_tokens": 5, "total_tokens": 60 }, "latencyMs": 716 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000202, "billedUsd": 0.0000202, "estimatedUsd": 0.0000202, "usage": { "prompt_tokens": 59, "completion_tokens": 1, "total_tokens": 60 }, "latencyMs": 458 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.0000179, "billedUsd": 0.0000179, "estimatedUsd": 0.0000179, "usage": { "prompt_tokens": 43, "completion_tokens": 2, "total_tokens": 45 }, "latencyMs": 577 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0000157, "billedUsd": 0.0000157, "estimatedUsd": 0.0000157, "usage": { "prompt_tokens": 44, "completion_tokens": 1, "total_tokens": 45 }, "latencyMs": 671 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000026099999999999997, "billedUsd": 0.0000261, "estimatedUsd": 0.000026099999999999997, "usage": { "prompt_tokens": 37, "completion_tokens": 6, "total_tokens": 43 }, "latencyMs": 473 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000164, "billedUsd": 0.0000164, "estimatedUsd": 0.0000164, "usage": { "prompt_tokens": 38, "completion_tokens": 2, "total_tokens": 40 }, "latencyMs": 705 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000026799999999999998, "billedUsd": 0.0000268, "estimatedUsd": 0.000026799999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 4, "total_tokens": 60 }, "latencyMs": 692 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000016599999999999997, "billedUsd": 0.0000166, "estimatedUsd": 0.000016599999999999997, "usage": { "prompt_tokens": 47, "completion_tokens": 1, "total_tokens": 48 }, "latencyMs": 412 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000016599999999999997, "billedUsd": 0.0000166, "estimatedUsd": 0.000016599999999999997, "usage": { "prompt_tokens": 47, "completion_tokens": 1, "total_tokens": 48 }, "latencyMs": 552 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0000367, "billedUsd": 0.0000367, "estimatedUsd": 0.0000367, "usage": { "prompt_tokens": 64, "completion_tokens": 7, "total_tokens": 71 }, "latencyMs": 479 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000393, "billedUsd": 0.0000393, "estimatedUsd": 0.0000393, "usage": { "prompt_tokens": 56, "completion_tokens": 9, "total_tokens": 65 }, "latencyMs": 504 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000017199999999999998, "billedUsd": 0.0000172, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 658 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000323, "billedUsd": 0.0000323, "estimatedUsd": 0.0000323, "usage": { "prompt_tokens": 91, "completion_tokens": 2, "total_tokens": 93 }, "latencyMs": 668 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000021999999999999996, "billedUsd": 0.000022, "estimatedUsd": 0.000021999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 1, "total_tokens": 66 }, "latencyMs": 438 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000277, "billedUsd": 0.0000277, "estimatedUsd": 0.0000277, "usage": { "prompt_tokens": 59, "completion_tokens": 4, "total_tokens": 63 }, "latencyMs": 506 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0001372, "billedUsd": 0.0001372, "estimatedUsd": 0.0001372, "usage": { "prompt_tokens": 74, "completion_tokens": 46, "total_tokens": 120 }, "latencyMs": 862 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000184, "billedUsd": 0.0000184, "estimatedUsd": 0.0000184, "usage": { "prompt_tokens": 53, "completion_tokens": 1, "total_tokens": 54 }, "latencyMs": 459 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000225, "billedUsd": 0.0000225, "estimatedUsd": 0.0000225, "usage": { "prompt_tokens": 50, "completion_tokens": 3, "total_tokens": 53 }, "latencyMs": 414 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0000228, "billedUsd": 0.0000228, "estimatedUsd": 0.0000228, "usage": { "prompt_tokens": 51, "completion_tokens": 3, "total_tokens": 54 }, "latencyMs": 456 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000265, "billedUsd": 0.0000265, "estimatedUsd": 0.0000265, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 422 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000047520000000000006, "billedUsd": 0.00007917, "estimatedUsd": 0.000047520000000000006, "usage": { "prompt_tokens": 66, "completion_tokens": 2, "total_tokens": 68 }, "latencyMs": 452 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00050358, "billedUsd": 0.00090502, "estimatedUsd": 0.00050358, "usage": { "prompt_tokens": 82, "completion_tokens": 227, "total_tokens": 309 }, "latencyMs": 15530 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00016896, "billedUsd": 0.00032052, "estimatedUsd": 0.00016896, "usage": { "prompt_tokens": 58, "completion_tokens": 66, "total_tokens": 124 }, "latencyMs": 2872 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00023099999999999998, "billedUsd": 0.000415898, "estimatedUsd": 0.00023099999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 98, "total_tokens": 154 }, "latencyMs": 1071 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00034518, "billedUsd": 0.000415512, "estimatedUsd": 0.00034518, "usage": { "prompt_tokens": 148, "completion_tokens": 125, "total_tokens": 273 }, "latencyMs": 3614 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0002673, "billedUsd": 0.00045199206, "estimatedUsd": 0.0002673, "usage": { "prompt_tokens": 63, "completion_tokens": 114, "total_tokens": 177 }, "latencyMs": 3783 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00017622, "billedUsd": 0.00034626, "estimatedUsd": 0.00017622, "usage": { "prompt_tokens": 63, "completion_tokens": 68, "total_tokens": 131 }, "latencyMs": 1013 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00004686, "billedUsd": 0.000070992, "estimatedUsd": 0.00004686, "usage": { "prompt_tokens": 62, "completion_tokens": 3, "total_tokens": 65 }, "latencyMs": 2620 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0001452, "billedUsd": 0.000200187, "estimatedUsd": 0.0001452, "usage": { "prompt_tokens": 91, "completion_tokens": 43, "total_tokens": 134 }, "latencyMs": 1254 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00031020000000000006, "billedUsd": 0.00029982346, "estimatedUsd": 0.00031020000000000006, "usage": { "prompt_tokens": 77, "completion_tokens": 131, "total_tokens": 208 }, "latencyMs": 4173 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00017225999999999998, "billedUsd": 0.00015842352, "estimatedUsd": 0.00017225999999999998, "usage": { "prompt_tokens": 66, "completion_tokens": 65, "total_tokens": 131 }, "latencyMs": 3531 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0001419, "billedUsd": 0.0002197, "estimatedUsd": 0.0001419, "usage": { "prompt_tokens": 77, "completion_tokens": 46, "total_tokens": 123 }, "latencyMs": 1381 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00010296, "billedUsd": 0.0001131, "estimatedUsd": 0.00010296, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 993 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00030822, "billedUsd": 0.000302168545, "estimatedUsd": 0.00030822, "usage": { "prompt_tokens": 65, "completion_tokens": 134, "total_tokens": 199 }, "latencyMs": 4000 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00004884, "billedUsd": 0.000080301, "estimatedUsd": 0.00004884, "usage": { "prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68 }, "latencyMs": 678 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00019668, "billedUsd": 0.0003828, "estimatedUsd": 0.00019668, "usage": { "prompt_tokens": 64, "completion_tokens": 78, "total_tokens": 142 }, "latencyMs": 1510 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00014124, "billedUsd": 0.00012593975, "estimatedUsd": 0.00014124, "usage": { "prompt_tokens": 61, "completion_tokens": 51, "total_tokens": 112 }, "latencyMs": 2203 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0001584, "billedUsd": 0.0002781972, "estimatedUsd": 0.0001584, "usage": { "prompt_tokens": 60, "completion_tokens": 60, "total_tokens": 120 }, "latencyMs": 2614 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0001584, "billedUsd": 0.00021402, "estimatedUsd": 0.0001584, "usage": { "prompt_tokens": 135, "completion_tokens": 35, "total_tokens": 170 }, "latencyMs": 2612 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00017556, "billedUsd": 0.00032955, "estimatedUsd": 0.00017556, "usage": { "prompt_tokens": 53, "completion_tokens": 71, "total_tokens": 124 }, "latencyMs": 2157 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model extraction correct, failed arithmetic, parsing fit.", "spendUsd": 0.00101772, "billedUsd": 0.0018357, "estimatedUsd": 0.00101772, "usage": { "prompt_tokens": 81, "completion_tokens": 487, "total_tokens": 568 }, "latencyMs": 5430 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00033066, "billedUsd": 0.00031337284, "estimatedUsd": 0.00033066, "usage": { "prompt_tokens": 87, "completion_tokens": 138, "total_tokens": 225 }, "latencyMs": 4455 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0000693, "billedUsd": 0.00008265, "estimatedUsd": 0.0000693, "usage": { "prompt_tokens": 75, "completion_tokens": 10, "total_tokens": 85 }, "latencyMs": 1083 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.00005478, "billedUsd": 0.000088218, "estimatedUsd": 0.00005478, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 26656 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00012342, "billedUsd": 0.00013224, "estimatedUsd": 0.00012342, "usage": { "prompt_tokens": 82, "completion_tokens": 35, "total_tokens": 117 }, "latencyMs": 11707 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00025278, "billedUsd": 0.0004708, "estimatedUsd": 0.00025278, "usage": { "prompt_tokens": 71, "completion_tokens": 104, "total_tokens": 175 }, "latencyMs": 3265 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00005676, "billedUsd": 0.0001066, "estimatedUsd": 0.00005676, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 1226 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00050688, "billedUsd": 0.000754728, "estimatedUsd": 0.00050688, "usage": { "prompt_tokens": 63, "completion_tokens": 235, "total_tokens": 298 }, "latencyMs": 4841 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0004356, "billedUsd": 0.00072531636, "estimatedUsd": 0.0004356, "usage": { "prompt_tokens": 78, "completion_tokens": 194, "total_tokens": 272 }, "latencyMs": 4962 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000050820000000000005, "billedUsd": 0.000080301, "estimatedUsd": 0.000050820000000000005, "usage": { "prompt_tokens": 59, "completion_tokens": 6, "total_tokens": 65 }, "latencyMs": 1741 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00020658, "billedUsd": 0.00019663972, "estimatedUsd": 0.00020658, "usage": { "prompt_tokens": 61, "completion_tokens": 84, "total_tokens": 145 }, "latencyMs": 8315 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Grok", "spendUsd": 0.00003762, "billedUsd": 0.000061074, "estimatedUsd": 0.00003762, "usage": { "prompt_tokens": 48, "completion_tokens": 3, "total_tokens": 51 }, "latencyMs": 1290 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00012738000000000002, "billedUsd": 0.000178524, "estimatedUsd": 0.00012738000000000002, "usage": { "prompt_tokens": 127, "completion_tokens": 22, "total_tokens": 149 }, "latencyMs": 3271 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00014387999999999998, "billedUsd": 0.000262409, "estimatedUsd": 0.00014387999999999998, "usage": { "prompt_tokens": 41, "completion_tokens": 59, "total_tokens": 100 }, "latencyMs": 1188 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00018216, "billedUsd": 0.000176749925, "estimatedUsd": 0.00018216, "usage": { "prompt_tokens": 42, "completion_tokens": 78, "total_tokens": 120 }, "latencyMs": 2917 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00005016000000000001, "billedUsd": 0.00006177, "estimatedUsd": 0.00005016000000000001, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1401 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00003762, "billedUsd": 0.000062205, "estimatedUsd": 0.00003762, "usage": { "prompt_tokens": 51, "completion_tokens": 2, "total_tokens": 53 }, "latencyMs": 695 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00034914, "billedUsd": 0.0006438, "estimatedUsd": 0.00034914, "usage": { "prompt_tokens": 52, "completion_tokens": 159, "total_tokens": 211 }, "latencyMs": 2817 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0002343, "billedUsd": 0.0004528, "estimatedUsd": 0.0002343, "usage": { "prompt_tokens": 139, "completion_tokens": 72, "total_tokens": 211 }, "latencyMs": 2082 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000594, "billedUsd": 0.000104, "estimatedUsd": 0.0000594, "usage": { "prompt_tokens": 60, "completion_tokens": 10, "total_tokens": 70 }, "latencyMs": 927 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0001452, "billedUsd": 0.000232224, "estimatedUsd": 0.0001452, "usage": { "prompt_tokens": 52, "completion_tokens": 56, "total_tokens": 108 }, "latencyMs": 2628 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00037158, "billedUsd": 0.000666759, "estimatedUsd": 0.00037158, "usage": { "prompt_tokens": 86, "completion_tokens": 159, "total_tokens": 245 }, "latencyMs": 8242 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000047520000000000006, "billedUsd": 0.0000609, "estimatedUsd": 0.000047520000000000006, "usage": { "prompt_tokens": 66, "completion_tokens": 2, "total_tokens": 68 }, "latencyMs": 546 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000046200000000000005, "billedUsd": 0.000075777, "estimatedUsd": 0.000046200000000000005, "usage": { "prompt_tokens": 61, "completion_tokens": 3, "total_tokens": 64 }, "latencyMs": 1578 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.00011286000000000001, "billedUsd": 0.000156078, "estimatedUsd": 0.00011286000000000001, "usage": { "prompt_tokens": 72, "completion_tokens": 33, "total_tokens": 105 }, "latencyMs": 3607 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00021648000000000001, "billedUsd": 0.0004224, "estimatedUsd": 0.00021648000000000001, "usage": { "prompt_tokens": 136, "completion_tokens": 64, "total_tokens": 200 }, "latencyMs": 2087 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00004158, "billedUsd": 0.000066729, "estimatedUsd": 0.00004158, "usage": { "prompt_tokens": 51, "completion_tokens": 4, "total_tokens": 55 }, "latencyMs": 553 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0000396, "billedUsd": 0.00005046, "estimatedUsd": 0.0000396, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 787 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00004884, "billedUsd": 0.000078039, "estimatedUsd": 0.00004884, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 2952 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00005148, "billedUsd": 0.000015771868, "estimatedUsd": 0.00005148, "usage": { "prompt_tokens": 66, "completion_tokens": 56, "total_tokens": 122 }, "latencyMs": 4797 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0000935, "billedUsd": 0.00004557, "estimatedUsd": 0.0000935, "usage": { "prompt_tokens": 161, "completion_tokens": 88, "total_tokens": 249 }, "latencyMs": 2003 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00004576, "billedUsd": 0.000021754, "estimatedUsd": 0.00004576, "usage": { "prompt_tokens": 58, "completion_tokens": 50, "total_tokens": 108 }, "latencyMs": 1192 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000048620000000000005, "billedUsd": 0.000018592, "estimatedUsd": 0.000048620000000000005, "usage": { "prompt_tokens": 56, "completion_tokens": 55, "total_tokens": 111 }, "latencyMs": 3629 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00006886, "billedUsd": 0.00003464, "estimatedUsd": 0.00006886, "usage": { "prompt_tokens": 148, "completion_tokens": 55, "total_tokens": 203 }, "latencyMs": 1325 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00001518, "billedUsd": 0.0000046137, "estimatedUsd": 0.00001518, "usage": { "prompt_tokens": 63, "completion_tokens": 2, "total_tokens": 65 }, "latencyMs": 1891 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000049279999999999996, "billedUsd": 0.0000238, "estimatedUsd": 0.000049279999999999996, "usage": { "prompt_tokens": 62, "completion_tokens": 54, "total_tokens": 116 }, "latencyMs": 951 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00003674, "billedUsd": 0.00001848, "estimatedUsd": 0.00003674, "usage": { "prompt_tokens": 62, "completion_tokens": 35, "total_tokens": 97 }, "latencyMs": 1408 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.00011374, "billedUsd": 0.000042, "estimatedUsd": 0.00011374, "usage": { "prompt_tokens": 91, "completion_tokens": 142, "total_tokens": 233 }, "latencyMs": 2889 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0001067, "billedUsd": 0.00008, "estimatedUsd": 0.0001067, "usage": { "prompt_tokens": 77, "completion_tokens": 136, "total_tokens": 213 }, "latencyMs": 2271 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00008184, "billedUsd": 0.0000378, "estimatedUsd": 0.00008184, "usage": { "prompt_tokens": 66, "completion_tokens": 102, "total_tokens": 168 }, "latencyMs": 2048 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0000473, "billedUsd": 0.0000129563, "estimatedUsd": 0.0000473, "usage": { "prompt_tokens": 77, "completion_tokens": 46, "total_tokens": 123 }, "latencyMs": 6955 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00009834000000000001, "billedUsd": 0.000028708344, "estimatedUsd": 0.00009834000000000001, "usage": { "prompt_tokens": 78, "completion_tokens": 123, "total_tokens": 201 }, "latencyMs": 3245 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.000077, "billedUsd": 0.000043, "estimatedUsd": 0.000077, "usage": { "prompt_tokens": 65, "completion_tokens": 95, "total_tokens": 160 }, "latencyMs": 2695 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00005258, "billedUsd": 0.000020272, "estimatedUsd": 0.00005258, "usage": { "prompt_tokens": 65, "completion_tokens": 58, "total_tokens": 123 }, "latencyMs": 2567 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected launch" }, "output": "pricing", "spendUsd": 0.00001584, "billedUsd": 0.0000047817, "estimatedUsd": 0.00001584, "usage": { "prompt_tokens": 63, "completion_tokens": 3, "total_tokens": 66 }, "latencyMs": 1603 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000055660000000000006, "billedUsd": 0.00002646, "estimatedUsd": 0.000055660000000000006, "usage": { "prompt_tokens": 61, "completion_tokens": 64, "total_tokens": 125 }, "latencyMs": 1934 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000050820000000000005, "billedUsd": 0.000019488, "estimatedUsd": 0.000050820000000000005, "usage": { "prompt_tokens": 60, "completion_tokens": 57, "total_tokens": 117 }, "latencyMs": 2079 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000013640000000000002, "billedUsd": 0.0000054, "estimatedUsd": 0.000013640000000000002, "usage": { "prompt_tokens": 56, "completion_tokens": 2, "total_tokens": 58 }, "latencyMs": 529 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00006248000000000001, "billedUsd": 0.000028489, "estimatedUsd": 0.00006248000000000001, "usage": { "prompt_tokens": 53, "completion_tokens": 77, "total_tokens": 130 }, "latencyMs": 3287 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "missing": [], "constraintFailures": [ "wordCount=7, expected=8" ], "wordCount": 7, "sentenceCount": 1 }, "output": "Cheaper extraction succeeded but multi-step arithmetic failed.", "spendUsd": 0.000025080000000000004, "billedUsd": 0.0000073479, "estimatedUsd": 0.000025080000000000004, "usage": { "prompt_tokens": 81, "completion_tokens": 11, "total_tokens": 92 }, "latencyMs": 2020 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000026399999999999998, "billedUsd": 0.00000981, "estimatedUsd": 0.000026399999999999998, "usage": { "prompt_tokens": 87, "completion_tokens": 11, "total_tokens": 98 }, "latencyMs": 1037 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.00010229999999999999, "billedUsd": 0.00005675, "estimatedUsd": 0.00010229999999999999, "usage": { "prompt_tokens": 75, "completion_tokens": 130, "total_tokens": 205 }, "latencyMs": 2871 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost per Correct Answer", "spendUsd": 0.0001232, "billedUsd": 0.000054484, "estimatedUsd": 0.0001232, "usage": { "prompt_tokens": 68, "completion_tokens": 164, "total_tokens": 232 }, "latencyMs": 4731 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00006688, "billedUsd": 0.00002037938, "estimatedUsd": 0.00006688, "usage": { "prompt_tokens": 82, "completion_tokens": 74, "total_tokens": 156 }, "latencyMs": 2789 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00008888, "billedUsd": 0.00004102, "estimatedUsd": 0.00008888, "usage": { "prompt_tokens": 71, "completion_tokens": 111, "total_tokens": 182 }, "latencyMs": 2204 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0000836, "billedUsd": 0.0000625, "estimatedUsd": 0.0000836, "usage": { "prompt_tokens": 74, "completion_tokens": 102, "total_tokens": 176 }, "latencyMs": 3776 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00014124000000000002, "billedUsd": 0.00006286, "estimatedUsd": 0.00014124000000000002, "usage": { "prompt_tokens": 63, "completion_tokens": 193, "total_tokens": 256 }, "latencyMs": 2396 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00015312, "billedUsd": 0.00005488, "estimatedUsd": 0.00015312, "usage": { "prompt_tokens": 78, "completion_tokens": 206, "total_tokens": 284 }, "latencyMs": 3570 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 3-6-3" }, "output": "3-5-3", "spendUsd": 0.00001694, "billedUsd": 0.0000050141, "estimatedUsd": 0.00001694, "usage": { "prompt_tokens": 59, "completion_tokens": 6, "total_tokens": 65 }, "latencyMs": 2028 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000014740000000000001, "billedUsd": 0.00000585, "estimatedUsd": 0.000014740000000000001, "usage": { "prompt_tokens": 61, "completion_tokens": 2, "total_tokens": 63 }, "latencyMs": 719 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00004686, "billedUsd": 0.00002645, "estimatedUsd": 0.00004686, "usage": { "prompt_tokens": 48, "completion_tokens": 55, "total_tokens": 103 }, "latencyMs": 1862 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00003762, "billedUsd": 0.00001742, "estimatedUsd": 0.00003762, "usage": { "prompt_tokens": 48, "completion_tokens": 41, "total_tokens": 89 }, "latencyMs": 1251 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00004268, "billedUsd": 0.00002002, "estimatedUsd": 0.00004268, "usage": { "prompt_tokens": 41, "completion_tokens": 51, "total_tokens": 92 }, "latencyMs": 1632 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000891, "billedUsd": 0.0000675, "estimatedUsd": 0.0000891, "usage": { "prompt_tokens": 42, "completion_tokens": 121, "total_tokens": 163 }, "latencyMs": 3764 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000016720000000000003, "billedUsd": 0.0000049819, "estimatedUsd": 0.000016720000000000003, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 3094 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000060939999999999996, "billedUsd": 0.00003192, "estimatedUsd": 0.000060939999999999996, "usage": { "prompt_tokens": 130, "completion_tokens": 49, "total_tokens": 179 }, "latencyMs": 2434 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0000693, "billedUsd": 0.000020113562, "estimatedUsd": 0.0000693, "usage": { "prompt_tokens": 51, "completion_tokens": 88, "total_tokens": 139 }, "latencyMs": 6818 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0000528, "billedUsd": 0.00002412, "estimatedUsd": 0.0000528, "usage": { "prompt_tokens": 60, "completion_tokens": 60, "total_tokens": 120 }, "latencyMs": 1235 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000792, "billedUsd": 0.00003484, "estimatedUsd": 0.0000792, "usage": { "prompt_tokens": 60, "completion_tokens": 100, "total_tokens": 160 }, "latencyMs": 1665 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00006116, "billedUsd": 0.00003206, "estimatedUsd": 0.00006116, "usage": { "prompt_tokens": 131, "completion_tokens": 49, "total_tokens": 180 }, "latencyMs": 1920 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000020900000000000003, "billedUsd": 0.00001974, "estimatedUsd": 0.000020900000000000003, "usage": { "prompt_tokens": 86, "completion_tokens": 3, "total_tokens": 89 }, "latencyMs": 6999 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00013089999999999998, "billedUsd": 0.00006085, "estimatedUsd": 0.00013089999999999998, "usage": { "prompt_tokens": 145, "completion_tokens": 150, "total_tokens": 295 }, "latencyMs": 2887 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00006358000000000001, "billedUsd": 0.00002982, "estimatedUsd": 0.00006358000000000001, "usage": { "prompt_tokens": 61, "completion_tokens": 76, "total_tokens": 137 }, "latencyMs": 1713 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.00003762, "billedUsd": 0.00001242, "estimatedUsd": 0.00003762, "usage": { "prompt_tokens": 72, "completion_tokens": 33, "total_tokens": 105 }, "latencyMs": 1232 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000054120000000000004, "billedUsd": 0.000024522, "estimatedUsd": 0.000054120000000000004, "usage": { "prompt_tokens": 57, "completion_tokens": 63, "total_tokens": 120 }, "latencyMs": 1681 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00001386, "billedUsd": 0.00001295, "estimatedUsd": 0.00001386, "usage": { "prompt_tokens": 51, "completion_tokens": 4, "total_tokens": 55 }, "latencyMs": 518 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000013199999999999999, "billedUsd": 0.0000040026, "estimatedUsd": 0.000013199999999999999, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 2572 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000891, "billedUsd": 0.00004286, "estimatedUsd": 0.0000891, "usage": { "prompt_tokens": 138, "completion_tokens": 89, "total_tokens": 227 }, "latencyMs": 2186 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00043940000000000006, "billedUsd": 0.0002197, "estimatedUsd": 0.00043940000000000006, "usage": { "prompt_tokens": 75, "completion_tokens": 76, "total_tokens": 151 }, "latencyMs": 981 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00048820000000000005, "billedUsd": 0.0004882, "estimatedUsd": 0.00048820000000000005, "usage": { "prompt_tokens": 91, "completion_tokens": 82, "total_tokens": 173 }, "latencyMs": 726 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0002904, "billedUsd": 0.0002904, "estimatedUsd": 0.0002904, "usage": { "prompt_tokens": 66, "completion_tokens": 45, "total_tokens": 111 }, "latencyMs": 932 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0005398, "billedUsd": 0.0002699, "estimatedUsd": 0.0005398, "usage": { "prompt_tokens": 65, "completion_tokens": 102, "total_tokens": 167 }, "latencyMs": 1109 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0005404, "billedUsd": 0.0005404, "estimatedUsd": 0.0005404, "usage": { "prompt_tokens": 78, "completion_tokens": 98, "total_tokens": 176 }, "latencyMs": 733 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000161, "billedUsd": 0.000161, "estimatedUsd": 0.000161, "usage": { "prompt_tokens": 71, "completion_tokens": 14, "total_tokens": 85 }, "latencyMs": 447 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00028419999999999997, "billedUsd": 0.0001421, "estimatedUsd": 0.00028419999999999997, "usage": { "prompt_tokens": 71, "completion_tokens": 42, "total_tokens": 113 }, "latencyMs": 781 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00040300000000000004, "billedUsd": 0.000403, "estimatedUsd": 0.00040300000000000004, "usage": { "prompt_tokens": 71, "completion_tokens": 69, "total_tokens": 140 }, "latencyMs": 657 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0018192000000000002, "billedUsd": 0.0009096, "estimatedUsd": 0.0018192000000000002, "usage": { "prompt_tokens": 102, "completion_tokens": 381, "total_tokens": 483 }, "latencyMs": 6299 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.00082, "billedUsd": 0.00041, "estimatedUsd": 0.00082, "usage": { "prompt_tokens": 86, "completion_tokens": 159, "total_tokens": 245 }, "latencyMs": 1966 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0003412, "billedUsd": 0.0001706, "estimatedUsd": 0.0003412, "usage": { "prompt_tokens": 74, "completion_tokens": 54, "total_tokens": 128 }, "latencyMs": 995 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0002832, "billedUsd": 0.0002832, "estimatedUsd": 0.0002832, "usage": { "prompt_tokens": 86, "completion_tokens": 37, "total_tokens": 123 }, "latencyMs": 555 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0005576000000000001, "billedUsd": 0.0005576, "estimatedUsd": 0.0005576000000000001, "usage": { "prompt_tokens": 84, "completion_tokens": 100, "total_tokens": 184 }, "latencyMs": 694 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0009778, "billedUsd": 0.0009778, "estimatedUsd": 0.0009778, "usage": { "prompt_tokens": 73, "completion_tokens": 199, "total_tokens": 272 }, "latencyMs": 1736 }, { "taskId": "classify-001", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0002458, "billedUsd": 0.0002458, "estimatedUsd": 0.0002458, "usage": { "prompt_tokens": 75, "completion_tokens": 32, "total_tokens": 107 }, "latencyMs": 884 }, { "taskId": "classify-002", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0012200000000000002, "billedUsd": 0.00061, "estimatedUsd": 0.0012200000000000002, "usage": { "prompt_tokens": 70, "completion_tokens": 255, "total_tokens": 325 }, "latencyMs": 5526 }, { "taskId": "classify-003", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00046320000000000004, "billedUsd": 0.0002316, "estimatedUsd": 0.00046320000000000004, "usage": { "prompt_tokens": 70, "completion_tokens": 83, "total_tokens": 153 }, "latencyMs": 1370 }, { "taskId": "classify-004", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0013212, "billedUsd": 0.0006606, "estimatedUsd": 0.0013212, "usage": { "prompt_tokens": 70, "completion_tokens": 278, "total_tokens": 348 }, "latencyMs": 2285 }, { "taskId": "classify-005", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00010279999999999999, "billedUsd": 0.0001028, "estimatedUsd": 0.00010279999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 3, "total_tokens": 67 }, "latencyMs": 810 }, { "taskId": "classify-006", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.0004828, "billedUsd": 0.0002414, "estimatedUsd": 0.0004828, "usage": { "prompt_tokens": 62, "completion_tokens": 90, "total_tokens": 152 }, "latencyMs": 1518 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded at extraction but failed arithmetic.", "spendUsd": 0.0018038000000000002, "billedUsd": 0.001244622, "estimatedUsd": 0.0018038000000000002, "usage": { "prompt_tokens": 91, "completion_tokens": 381, "total_tokens": 472 }, "latencyMs": 5274 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000438, "billedUsd": 0.000219, "estimatedUsd": 0.000438, "usage": { "prompt_tokens": 96, "completion_tokens": 69, "total_tokens": 165 }, "latencyMs": 1166 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity compounds cost, reducing accuracy-per-dollar returns.", "spendUsd": 0.0024776000000000004, "billedUsd": 0.0012388, "estimatedUsd": 0.0024776000000000004, "usage": { "prompt_tokens": 82, "completion_tokens": 537, "total_tokens": 619 }, "latencyMs": 4095 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct AI Answer", "spendUsd": 0.0014014, "billedUsd": 0.0014014, "estimatedUsd": 0.0014014, "usage": { "prompt_tokens": 77, "completion_tokens": 294, "total_tokens": 371 }, "latencyMs": 1974 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0015868000000000002, "billedUsd": 0.0007934, "estimatedUsd": 0.0015868000000000002, "usage": { "prompt_tokens": 90, "completion_tokens": 332, "total_tokens": 422 }, "latencyMs": 11382 }, { "taskId": "code-001", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00016779999999999999, "billedUsd": 0.0001678, "estimatedUsd": 0.00016779999999999999, "usage": { "prompt_tokens": 79, "completion_tokens": 13, "total_tokens": 92 }, "latencyMs": 452 }, { "taskId": "code-002", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0006488, "billedUsd": 0.0003244, "estimatedUsd": 0.0006488, "usage": { "prompt_tokens": 80, "completion_tokens": 122, "total_tokens": 202 }, "latencyMs": 2986 }, { "taskId": "code-003", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0011084000000000003, "billedUsd": 0.0005542, "estimatedUsd": 0.0011084000000000003, "usage": { "prompt_tokens": 72, "completion_tokens": 229, "total_tokens": 301 }, "latencyMs": 1931 }, { "taskId": "code-004", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0005386, "billedUsd": 0.0005386, "estimatedUsd": 0.0005386, "usage": { "prompt_tokens": 83, "completion_tokens": 96, "total_tokens": 179 }, "latencyMs": 792 }, { "taskId": "code-005", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0007392, "billedUsd": 0.0007392, "estimatedUsd": 0.0007392, "usage": { "prompt_tokens": 66, "completion_tokens": 147, "total_tokens": 213 }, "latencyMs": 980 }, { "taskId": "code-006", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0003708, "billedUsd": 0.0001854, "estimatedUsd": 0.0003708, "usage": { "prompt_tokens": 70, "completion_tokens": 62, "total_tokens": 132 }, "latencyMs": 1566 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0003218, "billedUsd": 0.0003218, "estimatedUsd": 0.0003218, "usage": { "prompt_tokens": 57, "completion_tokens": 55, "total_tokens": 112 }, "latencyMs": 533 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0001326, "billedUsd": 0.0001326, "estimatedUsd": 0.0001326, "usage": { "prompt_tokens": 57, "completion_tokens": 12, "total_tokens": 69 }, "latencyMs": 463 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0006024, "billedUsd": 0.0006024, "estimatedUsd": 0.0006024, "usage": { "prompt_tokens": 50, "completion_tokens": 121, "total_tokens": 171 }, "latencyMs": 701 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00029140000000000004, "billedUsd": 0.0001457, "estimatedUsd": 0.00029140000000000004, "usage": { "prompt_tokens": 51, "completion_tokens": 50, "total_tokens": 101 }, "latencyMs": 917 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.001047, "billedUsd": 0.001047, "estimatedUsd": 0.001047, "usage": { "prompt_tokens": 69, "completion_tokens": 216, "total_tokens": 285 }, "latencyMs": 1168 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0001632, "billedUsd": 0.0001632, "estimatedUsd": 0.0001632, "usage": { "prompt_tokens": 60, "completion_tokens": 18, "total_tokens": 78 }, "latencyMs": 466 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00037440000000000005, "billedUsd": 0.0003744, "estimatedUsd": 0.00037440000000000005, "usage": { "prompt_tokens": 60, "completion_tokens": 66, "total_tokens": 126 }, "latencyMs": 687 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00046619999999999995, "billedUsd": 0.0002331, "estimatedUsd": 0.00046619999999999995, "usage": { "prompt_tokens": 69, "completion_tokens": 84, "total_tokens": 153 }, "latencyMs": 9138 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00039139999999999997, "billedUsd": 0.0003914, "estimatedUsd": 0.00039139999999999997, "usage": { "prompt_tokens": 69, "completion_tokens": 67, "total_tokens": 136 }, "latencyMs": 571 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00037860000000000004, "billedUsd": 0.0003786, "estimatedUsd": 0.00037860000000000004, "usage": { "prompt_tokens": 63, "completion_tokens": 66, "total_tokens": 129 }, "latencyMs": 629 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0006434, "billedUsd": 0.0003217, "estimatedUsd": 0.0006434, "usage": { "prompt_tokens": 95, "completion_tokens": 116, "total_tokens": 211 }, "latencyMs": 9873 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0002942, "billedUsd": 0.0002942, "estimatedUsd": 0.0002942, "usage": { "prompt_tokens": 75, "completion_tokens": 43, "total_tokens": 118 }, "latencyMs": 533 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000563, "billedUsd": 0.0002815, "estimatedUsd": 0.000563, "usage": { "prompt_tokens": 69, "completion_tokens": 106, "total_tokens": 175 }, "latencyMs": 2808 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.000439, "billedUsd": 0.000439, "estimatedUsd": 0.000439, "usage": { "prompt_tokens": 81, "completion_tokens": 74, "total_tokens": 155 }, "latencyMs": 701 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0008712, "billedUsd": 0.0008712, "estimatedUsd": 0.0008712, "usage": { "prompt_tokens": 66, "completion_tokens": 177, "total_tokens": 243 }, "latencyMs": 1978 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0009640000000000001, "billedUsd": 0.000482, "estimatedUsd": 0.0009640000000000001, "usage": { "prompt_tokens": 60, "completion_tokens": 200, "total_tokens": 260 }, "latencyMs": 5397 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0003566, "billedUsd": 0.0001783, "estimatedUsd": 0.0003566, "usage": { "prompt_tokens": 63, "completion_tokens": 61, "total_tokens": 124 }, "latencyMs": 1031 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0013316, "billedUsd": 0.0006658, "estimatedUsd": 0.0013316, "usage": { "prompt_tokens": 68, "completion_tokens": 281, "total_tokens": 349 }, "latencyMs": 4459 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001202, "billedUsd": 0.00101, "estimatedUsd": 0.001202, "usage": { "prompt_tokens": 271, "completion_tokens": 110, "total_tokens": 381 }, "latencyMs": 2230 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0026680000000000002, "billedUsd": 0.002476, "estimatedUsd": 0.0026680000000000002, "usage": { "prompt_tokens": 287, "completion_tokens": 349, "total_tokens": 636 }, "latencyMs": 5410 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.001738, "billedUsd": 0.001546, "estimatedUsd": 0.001738, "usage": { "prompt_tokens": 263, "completion_tokens": 202, "total_tokens": 465 }, "latencyMs": 3711 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001402, "billedUsd": 0.00121, "estimatedUsd": 0.001402, "usage": { "prompt_tokens": 263, "completion_tokens": 146, "total_tokens": 409 }, "latencyMs": 9905 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0014759999999999999, "billedUsd": 0.001284, "estimatedUsd": 0.0014759999999999999, "usage": { "prompt_tokens": 276, "completion_tokens": 154, "total_tokens": 430 }, "latencyMs": 2806 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001382, "billedUsd": 0.00119, "estimatedUsd": 0.001382, "usage": { "prompt_tokens": 268, "completion_tokens": 141, "total_tokens": 409 }, "latencyMs": 2722 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.001398, "billedUsd": 0.001206, "estimatedUsd": 0.001398, "usage": { "prompt_tokens": 267, "completion_tokens": 144, "total_tokens": 411 }, "latencyMs": 2798 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0018599999999999999, "billedUsd": 0.001668, "estimatedUsd": 0.0018599999999999999, "usage": { "prompt_tokens": 267, "completion_tokens": 221, "total_tokens": 488 }, "latencyMs": 3962 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.3,\"outputPerM\":2.5,\"context\":1048576}", "spendUsd": 0.003138, "billedUsd": 0.002946, "estimatedUsd": 0.003138, "usage": { "prompt_tokens": 297, "completion_tokens": 424, "total_tokens": 721 }, "latencyMs": 6819 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.002158, "billedUsd": 0.001966, "estimatedUsd": 0.002158, "usage": { "prompt_tokens": 281, "completion_tokens": 266, "total_tokens": 547 }, "latencyMs": 4308 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.002108, "billedUsd": 0.001916, "estimatedUsd": 0.002108, "usage": { "prompt_tokens": 271, "completion_tokens": 261, "total_tokens": 532 }, "latencyMs": 5121 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0024259999999999998, "billedUsd": 0.002234, "estimatedUsd": 0.0024259999999999998, "usage": { "prompt_tokens": 286, "completion_tokens": 309, "total_tokens": 595 }, "latencyMs": 4809 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0023539999999999998, "billedUsd": 0.002162, "estimatedUsd": 0.0023539999999999998, "usage": { "prompt_tokens": 283, "completion_tokens": 298, "total_tokens": 581 }, "latencyMs": 4721 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.0017199999999999997, "billedUsd": 0.001528, "estimatedUsd": 0.0017199999999999997, "usage": { "prompt_tokens": 269, "completion_tokens": 197, "total_tokens": 466 }, "latencyMs": 2986 }, { "taskId": "classify-001", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.001716, "billedUsd": 0.001524, "estimatedUsd": 0.001716, "usage": { "prompt_tokens": 270, "completion_tokens": 196, "total_tokens": 466 }, "latencyMs": 3457 }, { "taskId": "classify-002", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0018279999999999998, "billedUsd": 0.001636, "estimatedUsd": 0.0018279999999999998, "usage": { "prompt_tokens": 266, "completion_tokens": 216, "total_tokens": 482 }, "latencyMs": 3684 }, { "taskId": "classify-003", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.001774, "billedUsd": 0.001582, "estimatedUsd": 0.001774, "usage": { "prompt_tokens": 266, "completion_tokens": 207, "total_tokens": 473 }, "latencyMs": 3606 }, { "taskId": "classify-004", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.001674, "billedUsd": 0.001482, "estimatedUsd": 0.001674, "usage": { "prompt_tokens": 267, "completion_tokens": 190, "total_tokens": 457 }, "latencyMs": 3144 }, { "taskId": "classify-005", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001246, "billedUsd": 0.001054, "estimatedUsd": 0.001246, "usage": { "prompt_tokens": 260, "completion_tokens": 121, "total_tokens": 381 }, "latencyMs": 2905 }, { "taskId": "classify-006", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0016979999999999999, "billedUsd": 0.001506, "estimatedUsd": 0.0016979999999999999, "usage": { "prompt_tokens": 258, "completion_tokens": 197, "total_tokens": 455 }, "latencyMs": 3632 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model did extraction well but failed arithmetic", "spendUsd": 0.008248, "billedUsd": 0.008248, "estimatedUsd": 0.008248, "usage": { "prompt_tokens": 287, "completion_tokens": 1279, "total_tokens": 1566 }, "latencyMs": 21616 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0023, "billedUsd": 0.002108, "estimatedUsd": 0.0023, "usage": { "prompt_tokens": 292, "completion_tokens": 286, "total_tokens": 578 }, "latencyMs": 4498 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "High verbosity compounds cost.", "spendUsd": 0.006646, "billedUsd": 0.006454, "estimatedUsd": 0.006646, "usage": { "prompt_tokens": 278, "completion_tokens": 1015, "total_tokens": 1293 }, "latencyMs": 16613 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost of Correct Answers", "spendUsd": 0.00546, "billedUsd": 0.00546, "estimatedUsd": 0.00546, "usage": { "prompt_tokens": 273, "completion_tokens": 819, "total_tokens": 1092 }, "latencyMs": 15323 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0031119999999999997, "billedUsd": 0.00292, "estimatedUsd": 0.0031119999999999997, "usage": { "prompt_tokens": 287, "completion_tokens": 423, "total_tokens": 710 }, "latencyMs": 6288 }, { "taskId": "code-001", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.001724, "billedUsd": 0.001724, "estimatedUsd": 0.001724, "usage": { "prompt_tokens": 280, "completion_tokens": 194, "total_tokens": 474 }, "latencyMs": 3553 }, { "taskId": "code-002", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0026399999999999996, "billedUsd": 0.00264, "estimatedUsd": 0.0026399999999999996, "usage": { "prompt_tokens": 279, "completion_tokens": 347, "total_tokens": 626 }, "latencyMs": 4990 }, { "taskId": "code-003", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00208, "billedUsd": 0.00208, "estimatedUsd": 0.00208, "usage": { "prompt_tokens": 269, "completion_tokens": 257, "total_tokens": 526 }, "latencyMs": 4966 }, { "taskId": "code-004", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.002382, "billedUsd": 0.00219, "estimatedUsd": 0.002382, "usage": { "prompt_tokens": 282, "completion_tokens": 303, "total_tokens": 585 }, "latencyMs": 4629 }, { "taskId": "code-005", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.001814, "billedUsd": 0.001622, "estimatedUsd": 0.001814, "usage": { "prompt_tokens": 262, "completion_tokens": 215, "total_tokens": 477 }, "latencyMs": 4163 }, { "taskId": "code-006", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.001394, "billedUsd": 0.001394, "estimatedUsd": 0.001394, "usage": { "prompt_tokens": 268, "completion_tokens": 143, "total_tokens": 411 }, "latencyMs": 2888 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00203, "billedUsd": 0.001838, "estimatedUsd": 0.00203, "usage": { "prompt_tokens": 253, "completion_tokens": 254, "total_tokens": 507 }, "latencyMs": 4353 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0013939999999999998, "billedUsd": 0.001202, "estimatedUsd": 0.0013939999999999998, "usage": { "prompt_tokens": 253, "completion_tokens": 148, "total_tokens": 401 }, "latencyMs": 2735 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0021780000000000002, "billedUsd": 0.001986, "estimatedUsd": 0.0021780000000000002, "usage": { "prompt_tokens": 246, "completion_tokens": 281, "total_tokens": 527 }, "latencyMs": 4903 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.001592, "billedUsd": 0.0014, "estimatedUsd": 0.001592, "usage": { "prompt_tokens": 247, "completion_tokens": 183, "total_tokens": 430 }, "latencyMs": 4024 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0018700000000000001, "billedUsd": 0.00187, "estimatedUsd": 0.0018700000000000001, "usage": { "prompt_tokens": 266, "completion_tokens": 223, "total_tokens": 489 }, "latencyMs": 4087 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.002018, "billedUsd": 0.001826, "estimatedUsd": 0.002018, "usage": { "prompt_tokens": 256, "completion_tokens": 251, "total_tokens": 507 }, "latencyMs": 4149 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00213, "billedUsd": 0.001938, "estimatedUsd": 0.00213, "usage": { "prompt_tokens": 255, "completion_tokens": 270, "total_tokens": 525 }, "latencyMs": 4766 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001552, "billedUsd": 0.00136, "estimatedUsd": 0.001552, "usage": { "prompt_tokens": 269, "completion_tokens": 169, "total_tokens": 438 }, "latencyMs": 3643 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00192, "billedUsd": 0.001728, "estimatedUsd": 0.00192, "usage": { "prompt_tokens": 264, "completion_tokens": 232, "total_tokens": 496 }, "latencyMs": 3679 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001502, "billedUsd": 0.00131, "estimatedUsd": 0.001502, "usage": { "prompt_tokens": 259, "completion_tokens": 164, "total_tokens": 423 }, "latencyMs": 3392 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.002146, "billedUsd": 0.001954, "estimatedUsd": 0.002146, "usage": { "prompt_tokens": 296, "completion_tokens": 259, "total_tokens": 555 }, "latencyMs": 4247 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.005978, "billedUsd": 0.005786, "estimatedUsd": 0.005978, "usage": { "prompt_tokens": 271, "completion_tokens": 906, "total_tokens": 1177 }, "latencyMs": 14990 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.001162, "billedUsd": 0.00097, "estimatedUsd": 0.001162, "usage": { "prompt_tokens": 266, "completion_tokens": 105, "total_tokens": 371 }, "latencyMs": 1875 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00231, "billedUsd": 0.002118, "estimatedUsd": 0.00231, "usage": { "prompt_tokens": 279, "completion_tokens": 292, "total_tokens": 571 }, "latencyMs": 4757 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00149, "billedUsd": 0.00149, "estimatedUsd": 0.00149, "usage": { "prompt_tokens": 262, "completion_tokens": 161, "total_tokens": 423 }, "latencyMs": 3075 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001284, "billedUsd": 0.001092, "estimatedUsd": 0.001284, "usage": { "prompt_tokens": 258, "completion_tokens": 128, "total_tokens": 386 }, "latencyMs": 2506 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0012439999999999999, "billedUsd": 0.001244, "estimatedUsd": 0.0012439999999999999, "usage": { "prompt_tokens": 259, "completion_tokens": 121, "total_tokens": 380 }, "latencyMs": 2486 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0022320000000000005, "billedUsd": 0.00204, "estimatedUsd": 0.0022320000000000005, "usage": { "prompt_tokens": 264, "completion_tokens": 284, "total_tokens": 548 }, "latencyMs": 4761 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001094, "billedUsd": 0.0008764, "estimatedUsd": 0.001094, "usage": { "prompt_tokens": 271, "completion_tokens": 92, "total_tokens": 363 }, "latencyMs": 1993 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001666, "billedUsd": 0.0014484, "estimatedUsd": 0.001666, "usage": { "prompt_tokens": 287, "completion_tokens": 182, "total_tokens": 469 }, "latencyMs": 3490 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.001126, "billedUsd": 0.0009084, "estimatedUsd": 0.001126, "usage": { "prompt_tokens": 263, "completion_tokens": 100, "total_tokens": 363 }, "latencyMs": 2108 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001132, "billedUsd": 0.001132, "estimatedUsd": 0.001132, "usage": { "prompt_tokens": 263, "completion_tokens": 101, "total_tokens": 364 }, "latencyMs": 2184 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0012360000000000001, "billedUsd": 0.0010184, "estimatedUsd": 0.0012360000000000001, "usage": { "prompt_tokens": 276, "completion_tokens": 114, "total_tokens": 390 }, "latencyMs": 2381 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001232, "billedUsd": 0.0010144, "estimatedUsd": 0.001232, "usage": { "prompt_tokens": 268, "completion_tokens": 116, "total_tokens": 384 }, "latencyMs": 2753 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.001116, "billedUsd": 0.0008984, "estimatedUsd": 0.001116, "usage": { "prompt_tokens": 267, "completion_tokens": 97, "total_tokens": 364 }, "latencyMs": 1931 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.001092, "billedUsd": 0.0008744, "estimatedUsd": 0.001092, "usage": { "prompt_tokens": 267, "completion_tokens": 93, "total_tokens": 360 }, "latencyMs": 1862 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0019320000000000001, "billedUsd": 0.0017144, "estimatedUsd": 0.0019320000000000001, "usage": { "prompt_tokens": 297, "completion_tokens": 223, "total_tokens": 520 }, "latencyMs": 3604 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.001324, "billedUsd": 0.0011064, "estimatedUsd": 0.001324, "usage": { "prompt_tokens": 281, "completion_tokens": 127, "total_tokens": 408 }, "latencyMs": 2441 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.001112, "billedUsd": 0.0008944, "estimatedUsd": 0.001112, "usage": { "prompt_tokens": 271, "completion_tokens": 95, "total_tokens": 366 }, "latencyMs": 1918 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.001406, "billedUsd": 0.001406, "estimatedUsd": 0.001406, "usage": { "prompt_tokens": 286, "completion_tokens": 139, "total_tokens": 425 }, "latencyMs": 2446 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0014780000000000001, "billedUsd": 0.001478, "estimatedUsd": 0.0014780000000000001, "usage": { "prompt_tokens": 283, "completion_tokens": 152, "total_tokens": 435 }, "latencyMs": 2861 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.001084, "billedUsd": 0.0008664, "estimatedUsd": 0.001084, "usage": { "prompt_tokens": 269, "completion_tokens": 91, "total_tokens": 360 }, "latencyMs": 1826 }, { "taskId": "classify-001", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000984, "billedUsd": 0.0007664, "estimatedUsd": 0.000984, "usage": { "prompt_tokens": 270, "completion_tokens": 74, "total_tokens": 344 }, "latencyMs": 2014 }, { "taskId": "classify-002", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.001006, "billedUsd": 0.001006, "estimatedUsd": 0.001006, "usage": { "prompt_tokens": 266, "completion_tokens": 79, "total_tokens": 345 }, "latencyMs": 1856 }, { "taskId": "classify-003", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00112, "billedUsd": 0.0009024, "estimatedUsd": 0.00112, "usage": { "prompt_tokens": 266, "completion_tokens": 98, "total_tokens": 364 }, "latencyMs": 2198 }, { "taskId": "classify-004", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0008159999999999999, "billedUsd": 0.0005984, "estimatedUsd": 0.0008159999999999999, "usage": { "prompt_tokens": 267, "completion_tokens": 47, "total_tokens": 314 }, "latencyMs": 1510 }, { "taskId": "classify-005", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0010179999999999998, "billedUsd": 0.0008004, "estimatedUsd": 0.0010179999999999998, "usage": { "prompt_tokens": 260, "completion_tokens": 83, "total_tokens": 343 }, "latencyMs": 7142 }, { "taskId": "classify-006", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000954, "billedUsd": 0.000954, "estimatedUsd": 0.000954, "usage": { "prompt_tokens": 258, "completion_tokens": 73, "total_tokens": 331 }, "latencyMs": 1654 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction succeeded but failed arithmetic reasoning tasks.", "spendUsd": 0.002848, "billedUsd": 0.0026304, "estimatedUsd": 0.002848, "usage": { "prompt_tokens": 287, "completion_tokens": 379, "total_tokens": 666 }, "latencyMs": 6250 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00119, "billedUsd": 0.0009724, "estimatedUsd": 0.00119, "usage": { "prompt_tokens": 292, "completion_tokens": 101, "total_tokens": 393 }, "latencyMs": 1878 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "High verbosity increases cost per answer.", "spendUsd": 0.002758, "billedUsd": 0.0025404, "estimatedUsd": 0.002758, "usage": { "prompt_tokens": 278, "completion_tokens": 367, "total_tokens": 645 }, "latencyMs": 6726 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Correct Answer Cost Comparison", "spendUsd": 0.001422, "billedUsd": 0.0012044, "estimatedUsd": 0.001422, "usage": { "prompt_tokens": 273, "completion_tokens": 146, "total_tokens": 419 }, "latencyMs": 3380 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0017980000000000001, "billedUsd": 0.0015804, "estimatedUsd": 0.0017980000000000001, "usage": { "prompt_tokens": 287, "completion_tokens": 204, "total_tokens": 491 }, "latencyMs": 4004 }, { "taskId": "code-001", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.001382, "billedUsd": 0.001382, "estimatedUsd": 0.001382, "usage": { "prompt_tokens": 280, "completion_tokens": 137, "total_tokens": 417 }, "latencyMs": 2818 }, { "taskId": "code-002", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0014759999999999999, "billedUsd": 0.0012584, "estimatedUsd": 0.0014759999999999999, "usage": { "prompt_tokens": 279, "completion_tokens": 153, "total_tokens": 432 }, "latencyMs": 2918 }, { "taskId": "code-003", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0011619999999999998, "billedUsd": 0.0009444, "estimatedUsd": 0.0011619999999999998, "usage": { "prompt_tokens": 269, "completion_tokens": 104, "total_tokens": 373 }, "latencyMs": 2432 }, { "taskId": "code-004", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001968, "billedUsd": 0.0017504, "estimatedUsd": 0.001968, "usage": { "prompt_tokens": 282, "completion_tokens": 234, "total_tokens": 516 }, "latencyMs": 3754 }, { "taskId": "code-005", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00125, "billedUsd": 0.0010324, "estimatedUsd": 0.00125, "usage": { "prompt_tokens": 262, "completion_tokens": 121, "total_tokens": 383 }, "latencyMs": 2818 }, { "taskId": "code-006", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00086, "billedUsd": 0.0006424, "estimatedUsd": 0.00086, "usage": { "prompt_tokens": 268, "completion_tokens": 54, "total_tokens": 322 }, "latencyMs": 1516 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0013520000000000001, "billedUsd": 0.0011344, "estimatedUsd": 0.0013520000000000001, "usage": { "prompt_tokens": 253, "completion_tokens": 141, "total_tokens": 394 }, "latencyMs": 2847 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000926, "billedUsd": 0.0007084, "estimatedUsd": 0.000926, "usage": { "prompt_tokens": 253, "completion_tokens": 70, "total_tokens": 323 }, "latencyMs": 1677 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001362, "billedUsd": 0.001362, "estimatedUsd": 0.001362, "usage": { "prompt_tokens": 246, "completion_tokens": 145, "total_tokens": 391 }, "latencyMs": 3105 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000902, "billedUsd": 0.0006844, "estimatedUsd": 0.000902, "usage": { "prompt_tokens": 247, "completion_tokens": 68, "total_tokens": 315 }, "latencyMs": 1609 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.001156, "billedUsd": 0.0009384, "estimatedUsd": 0.001156, "usage": { "prompt_tokens": 266, "completion_tokens": 104, "total_tokens": 370 }, "latencyMs": 2314 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.001124, "billedUsd": 0.001124, "estimatedUsd": 0.001124, "usage": { "prompt_tokens": 256, "completion_tokens": 102, "total_tokens": 358 }, "latencyMs": 2302 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.001578, "billedUsd": 0.0013604, "estimatedUsd": 0.001578, "usage": { "prompt_tokens": 255, "completion_tokens": 178, "total_tokens": 433 }, "latencyMs": 3664 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001078, "billedUsd": 0.001078, "estimatedUsd": 0.001078, "usage": { "prompt_tokens": 269, "completion_tokens": 90, "total_tokens": 359 }, "latencyMs": 2628 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00126, "billedUsd": 0.00126, "estimatedUsd": 0.00126, "usage": { "prompt_tokens": 264, "completion_tokens": 122, "total_tokens": 386 }, "latencyMs": 2758 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001064, "billedUsd": 0.001064, "estimatedUsd": 0.001064, "usage": { "prompt_tokens": 259, "completion_tokens": 91, "total_tokens": 350 }, "latencyMs": 2081 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0025900000000000003, "billedUsd": 0.0023724, "estimatedUsd": 0.0025900000000000003, "usage": { "prompt_tokens": 296, "completion_tokens": 333, "total_tokens": 629 }, "latencyMs": 5770 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.003374, "billedUsd": 0.0031564, "estimatedUsd": 0.003374, "usage": { "prompt_tokens": 271, "completion_tokens": 472, "total_tokens": 743 }, "latencyMs": 7897 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0011020000000000001, "billedUsd": 0.0008844, "estimatedUsd": 0.0011020000000000001, "usage": { "prompt_tokens": 266, "completion_tokens": 95, "total_tokens": 361 }, "latencyMs": 2000 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001542, "billedUsd": 0.001542, "estimatedUsd": 0.001542, "usage": { "prompt_tokens": 279, "completion_tokens": 164, "total_tokens": 443 }, "latencyMs": 3304 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001118, "billedUsd": 0.001118, "estimatedUsd": 0.001118, "usage": { "prompt_tokens": 262, "completion_tokens": 99, "total_tokens": 361 }, "latencyMs": 2269 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0010799999999999998, "billedUsd": 0.0008624, "estimatedUsd": 0.0010799999999999998, "usage": { "prompt_tokens": 258, "completion_tokens": 94, "total_tokens": 352 }, "latencyMs": 2402 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00113, "billedUsd": 0.00113, "estimatedUsd": 0.00113, "usage": { "prompt_tokens": 259, "completion_tokens": 102, "total_tokens": 361 }, "latencyMs": 2166 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.001296, "billedUsd": 0.0010784, "estimatedUsd": 0.001296, "usage": { "prompt_tokens": 264, "completion_tokens": 128, "total_tokens": 392 }, "latencyMs": 3574 } ] }